BufferScopeWorker.java

/*
** Module   : BufferScopeWorker.java
** Abstract : helper for rule set calculation of record scopes
**
** Copyright (c) 2005-2022, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ----------------Description-----------------
** 001 GES 20050921   @22789 First version which supports global and
**                           scoped data stored for each buffer name.
**                           Helper methods are provided to ease 
**                           management of the complex data structures
**                           used in buffer scope analysis.
** 002 ECF 20060406   @25399 Fix for limitation which prevents proper
**                           matching of buffers with unqualified fields.
**                           Introduced a more tolerant lookup algorithm
**                           for the getBuffer() user function.
** 003 ECF 20080730   @39262 Fixed memory leak. Implement finish() method
**                           to clean up when worker is terminated.
** 004 GES 20090518   @42377 Import change.
** 005 GES 20100819          Use a different map to provide a stable iteration order.
** 006 SVL 20130213          readOnly buffer property is handled.
** 007 CA  20130226          Buffer matching is always done using the explicit or implicit name.
** 008 SVL 20131115          Added support for forceDMOAlias and dynamicTable parameters.
** 009 OM  20161011          BufferScopeTracker.nearestRecordScopingBlock() signature changed.
**                           Added generics.
** 010 CA  20190128          Track buffer's static state (if OO).
** 011 CA  20190513          Fixed buffer usage from super-classes, in OO.
** 012 GES 20190620          Added multiple inheritance support (for OO interface defs).
** 013 CA  20200412          Added incremental conversion support.
** 014 CA  20201015          Replaced java.util.Stack with a non-synchronized custom implementation.
** 015 CA  20211214          The AST manager plugin must be context-local, for runtime conversion, as the 
**                           InMemoryRegistryPlugin is not thread-safe.
**                           All worker's state must be context-local, for runtime conversion to work in 
**                           multi-context mode, as the pattern workers are singletons.
**     CA  20220316          The lookup of the scope block in case of a free reference must be done using the 
**                           buffer's definition, and not the free reference block. 
**     CA  20220516          Allow re-initialization of the context-local state.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.convert;

import java.util.*;
import java.util.function.*;
import com.goldencode.util.Stack;
import com.goldencode.ast.*;
import com.goldencode.p2j.uast.*;
import com.goldencode.p2j.pattern.*;
import com.goldencode.p2j.security.*;


/**
 * A pattern worker whose purpose is to provide helpers for calculation of
 * all buffers, the blocks to which each buffer is scoped and the list of
 * AST nodes that reference each buffer.
 * <p>
 * The rules by which this scope calculation is handled are very complex.
 * Please see the <a href="../uast/package.html">UAST Package Summary</a>
 * for complete details.
 */
public class BufferScopeWorker
extends AbstractPatternWorker
implements ProgressParserTokenTypes
{
   /** Context local work area. */
   private static final ContextLocal<WorkArea> context = new ContextLocal<WorkArea>()
   {
      protected WorkArea initialValue()
      {
         return new WorkArea();
      }
   };

   /**
    * Default constructor which initializes libraries.
    */
   public BufferScopeWorker()
   {
      super();
      setLibrary(new BufferHelper());

      WorkArea wa = context.get();
      wa.allBufs = new LinkedHashMap<>();
      wa.blocks = new Stack<>();
   }
   
   /**
    * Clears state the first time each source file's AST is loaded.
    *
    * @param    ast
    *           AST about to be processed by the pattern engine.
    */
   public void visitAst(Aast ast)
   {
      WorkArea wa = context.get();
      
      // start fresh in a new file
      wa.allBufs.clear();
      
      // shouldn't be needed, but we will just be safe
      wa.blocks.clear();
   }
   
   /**
    * Hook to provide termination processing for a pattern worker.
    */
   public void finish()
   {
      WorkArea wa = context.get();

      wa.allBufs = null;
      wa.blocks = null;
      
      super.finish();
   }
   
   /**
    * Provides the core user-functions exported to pattern engine rules for
    * Progress buffer scope calculation.
    */
   public class BufferHelper
   {      
      /**
       * Get the IDs of the buffers or temp-tables defined in the super-classes for the specified
       * class.
       * 
       * @param     qname
       *            The qualified class name.
       *            
       * @return    A set of AST IDs.
       */
      public Iterator<Long> getSuperTables(String qname)
      {
         if (qname == null)
         {
            return Collections.emptyIterator();
         }
         
         // load all temp-tables from super-classes
         ClassDefinition clsDef = SymbolResolver.loadClassDefinition(qname);
         if (clsDef == null)
         {
            return Collections.emptyIterator();
         }
         
         Set<Long> res = new HashSet<>();
         
         Consumer<ClassDefinition> consumer = (ClassDefinition cls) ->
         {
            res.addAll(cls.getTableIds());
         };
         
         clsDef.processParentGraph(consumer);
         
         return res.iterator();
      }
      
      /**
       * Register this buffer name in the list of all possible buffers in
       * use in the current source file.  This should only be called from
       * a record reference (as opposed to a field reference).  Only the
       * first reference to a buffer will create the registration, subsequent
       * registrations will be ignored.  Each new registration creates an
       * instance of {@link BufferList}.
       *
       * @param    unique
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same.
       * @param    bufname
       *           The shortest unabbreviated unique buffer name.
       * @param    forceDMOAlias
       *           Explicit DMO alias for the converted buffer. Used for dynamic buffers.
       *           <code>null</code> if auto-converted alias should be used.
       * @param    schemaname
       *           The backing schema contruct that defines this node's
       *           structure.
       * @param    javaname
       *           The converted Java name for this buffer. <code>null</code>
       *           if this is an implicitly defined buffer.
       * @param    staticDef
       *           This is an explicit static buffer definition.
       * @param    dbname
       *           The logical database name for the buffer (this may be
       *           different from the database name in the schema parameter).
       * @param    implicit
       *           <code>true</code> if this buffer is being created due to
       *           an implicit record reference, <code>false</code> if the
       *           buffer was explicitly defined (e.g. with a DEFINE BUFFER
       *           statement).
       * @param    readOnly
       *           <code>true</code> if the buffer is read-only (like the OLD
       *           buffer of a database WRITE trigger).
       * @param    dynamicTable
       *           <code>true</code> if the buffer is for a dynamic table.
       * 
       * @return   The {@link BufferList} instance.
       */
      public BufferList registerBuffer(String  unique, 
                                       String  bufname,
                                       String  forceDMOAlias,
                                       String  schemaname,
                                       String  javaname,
                                       boolean staticDef,
                                       String  dbname,
                                       boolean implicit,
                                       boolean readOnly,
                                       boolean dynamicTable)
      {
         WorkArea wa = context.get();

         BufferList bl = wa.allBufs.get(unique);
         
         // is this a new buffer name?
         if (bl == null)
         {
            // create the BufferList and add it to the map
            bl = new BufferList(unique,
                                bufname,
                                forceDMOAlias,
                                schemaname,
                                javaname,
                                staticDef,
                                dbname,
                                implicit,
                                readOnly,
                                dynamicTable);
            wa.allBufs.put(unique, bl);
         }
         
         return bl;
      }
      
      /**
       * Access the master {@link BufferList} instance for the given buffer
       * name.  The returned object stores global state for the use of this
       * buffer name in the current Progress 4GL procedure.
       * <p>
       * Matching of <code>bufname</code> against the names of registered
       * BufferLists is performed in a progressively lenient manner.  The
       * first pass attempts to find an exact match on the full name.  If
       * this fails to produce a result, another match is attempted, this
       * time ignoring the database qualifier (the portion up to and including
       * the first dot).  If this fails to produce a result, another match is
       * attempted, this time ignoring the "uniqueness" qualifier (the
       * portion up to and including the first underscore).  If this fails,
       * <code>null</code> is returned.
       * <p>
       * Note:  this manner of matching is a workaround for a shortcoming of
       * the schema dictionary, which prevents an unqualified field from
       * being identified with an explicitly defined buffer rather than with
       * the table which backs it.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same.
       *
       * @return   The master for the given buffer name or <code>null</code> 
       *           if no such master exists.
       */
      public BufferList getBuffer(String bufname)
      {
         WorkArea wa = context.get();

         BufferList bl = wa.allBufs.get(bufname);
         
         if (bl == null)
         {
            // strip off database qualifier and compare remainder against
            // all buffer name keys, similarly adjusted
            bl = matchAfter(bufname, ".");
         }
         
         return bl;
      }
      
      /**
       * Returns an iterator to all {@link BufferList} instances, one
       * instance per unique buffer name.
       *
       * @return   The list of all unique buffers used in this source file.
       */
      public Iterator getBuffers()
      {
         WorkArea wa = context.get();

         return wa.allBufs.values().iterator();
      }
      
      /**
       * Push a new block onto the stack and maintain the block oriented
       * state for each buffer using the {@link BufferBlockState}.
       * <p>
       * If there is a bind target in the enclosing scope this bind target
       * is propagated into the new scope.  Likewise any pending rollup block
       * is propagated.  This must be done in order to ensure that relative
       * down movement below the pending rollup point will not lose track of
       * the fact that a rollup is needed at a higher level.  Other state
       * does not propagate "down" into the inner or nested block.
       * <p>
       * Due to a mismatch in location between where a weak or strong
       * reference occurs in the AST in relation to the block node to which
       * such references must be scoped, there may be a "staged" buffer
       * stored in the {@link BufferList} for each buffer name.  If present,
       * these staged scopes have their block nodes assigned and then it is
       * set as the active scope for the block.
       * <p>
       * The rules by which this scope calculation is handled are very
       * complex. Please see the <a href="../uast/package.html">UAST Package
       * Summary</a> for complete details.
       *
       * @param    node
       *           The AST node associated with this new scope.  Please note
       *           that this must be the same as the node to which a buffer
       *           would be scoped.
       */
      public void pushScope(Aast node)
      {
         WorkArea wa = context.get();

         // create the container for the block
         Map<String, BufferBlockState> lvl = new HashMap<>();
         
         // for each possible buffer, add a mapping
         for (String bufname : wa.allBufs.keySet())
         {
            // staged scope processing (this is deferred processing for
            // strong or weak scopes)
            BufferList master = getBuffer(bufname);
            BufferScopeTracker bst = master.getStaged();
            
            if (bst != null)
            {
               // if there is a staged buffer scope, set its block node, set
               // the buffer scope as the active scope for this block and
               // then clear the staged reference
               bst.setScopeBlock(node, master.getImplicit());
               master.setActive(bst);
               master.setStaged(null);
            }
            
            BufferBlockState inner = new BufferBlockState();
            
            // save the block AST node in the state instance
            inner.blockNode = node;
            
            BufferBlockState outer = getBlockState(bufname);
            
            // propagate state down (may still be null after this if it is
            // null in the enclosing block)
            if (outer != null)
            {
               inner.bindTarget        = outer.bindTarget;
               inner.pendingRollupNode = outer.pendingRollupNode;
            }
            
            lvl.put(bufname, inner);
         }
         
         // push the new scope onto the stack (must be done at the end!)
         wa.blocks.push(lvl);
      }
      
      /**
       * Pop the current block from the stack and maintain the block oriented
       * state for the next block based on the previous instances of 
       * {@link BufferBlockState} for each buffer.
       * <p>
       * Certain state from nested blocks must be propagated to outer nesting
       * levels during this operation.  This provides the ability for certain
       * deeply nested scopes to be used for "binding" or to trigger the
       * "rollup process".
       * <p>
       * If there is a pending rollup (because of a scope expansion that
       * occurred in the nested block) AND the target for that rollup is the
       * outer block, then the rollup process is invoked.
       * <p>
       * Nested scope expansions and first weak references can propagate "up"
       * to the outer block.  The scope expansions in particular, also
       * propagate into the bind target at enclosing levels.
       * <p>
       * If this is processing at the root node and there is a pending free
       * reference, the associated buffer scope is added to the master list
       * of buffers.
       * <p>
       * The rules by which this scope calculation is handled are very
       * complex. Please see the <a href="../uast/package.html">UAST Package
       * Summary</a> for complete details.
       *
       * @param    node
       *           The AST node associated with this scope.  Please note
       *           that this must be the same as the node to which a buffer
       *           would be scoped.
       */
      public void popScope(Aast node)
      {
         WorkArea wa = context.get();

         // check if this is the root node
         boolean root = node.isRoot();
         
         // pop our scope and obtain our map
         Map<String, BufferBlockState> lvl = wa.blocks.pop();
         
         Iterator keys = wa.allBufs.keySet().iterator();
         
         // for each possible buffer, propagate certain state to the next level
         while (keys.hasNext())
         {
            String           bufname = (String) keys.next();
            BufferList       master  = getBuffer(bufname);
            BufferBlockState inner   = lvl.get(bufname);
            
            // is this the root (external procedure) scope?
            if (root)
            {
               // is there a pending free reference?
               if (master.getPending() != null)
               {
                  // add this scope into the master list of buffers
                  master.addBufferScope(master.getPending());
               }
               
               // was there a previous expanded scope?
               if (inner.lastExpanded != null)
               {
                  // this previous expanded scope is a separate scope now
                  master.addBufferScope(inner.lastExpanded);
               }               
            }
              
            BufferBlockState outer = getBlockState(bufname);
            
            if (outer != null)
            {
               // check if the rollup process needs to be triggered (it only
               // triggers at the level to which the expansion was scoped)
               // this should be safe as the outer block node should never be null
               if (inner.pendingRollupNode == outer.blockNode)
               {
                  // in this case, there must be an active scope that is scoped to the outer
                  // block, this means there can be no items waiting in the rollup list of the
                  // inner block so we don't try to finalize those
                  
                  // time to rollup!
                  rollupBuffers(bufname, outer);
                  
                  // an expansion occurred, propagate this (and clear other state)
                  setExpanded(master, outer, inner.lastExpanded);
               }
               else
               {
                  // no rollup needed at this level, just propagate state up
                  outer.pendingRollupNode = inner.pendingRollupNode;
                  
                  // check if the nested block triggered an expansion
                  // (directly or via a sub-nested block)
                  if (inner.lastExpanded != null)
                  {
                     // clear our rollup list since the expanded scope acts like a firewall
                     finalizeBuffers(bufname, outer);
                     
                     // an expansion occurred, propagate this (and clear other state)
                     setExpanded(master, outer, inner.lastExpanded);
                  }
                  else
                  {
                     // expansions take precedence over propagation of the
                     // first weak reference, however at this point we know
                     // there was no expansion
                     if (inner.firstWeak != null)
                     {
                        // NO first weak reference in outer block?
                        if (outer.firstWeak == null)
                        {
                           // propagate the first weak ref up IF and ONLY IF
                           // there was a nested weak reference AND there was
                           // no prior first weak reference at this level
                           outer.firstWeak = inner.firstWeak;
                        }
                        
                        inner.rollupList.remove(inner.firstWeak);
                        outer.rollupList.add(inner.firstWeak);
                        outer.bindTarget = inner.firstWeak;
                     }
                  }
                  
                  // the inner block's rollup list needs to be finalized
                  // since it will NOT be otherwise rolled up and since it
                  // can't have been rolled up previously due to the lack
                  // of an expanded scope
                  finalizeBuffers(bufname, inner);
               }
            }
            else
            {
               // this is the root node so we have to finalize the rollup list
               finalizeBuffers(bufname, inner);
            }
            
            // is there is a current active scope?
            if (master.getActive() != null)
            {
               // is that active scope scoped to this block?
               if (node == master.getActive().getScopeBlock())
               {
                  // clear the active scope
                  master.setActive(null);
               }
            }
         }
      }
      
      /**
       * Accesses the block state of the named buffer from the current block.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same.
       *
       * @return   The block state for that buffer in the current block or
       *           <code>null</code> if this is called at the root node.
       */
      public BufferBlockState getBlockState(String bufname)
      {
         WorkArea wa = context.get();

         if (wa.blocks.empty())
         {
            return null;
         }
         
         // obtain the map for the current scope
         Map<String, BufferBlockState> lvl = wa.blocks.peek();
         
         return lvl.get(bufname);
      }
      
      /**
       * Accesses the master list of all buffers for this buffer name.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same.
       *
       * @return   The master list of all buffers for this buffer name.
       *
       * @throws   IllegalArgumentException
       *           If the buffer name is invalid.
       */
      public BufferList getMasterList(String bufname)
      {
         BufferList master = getBuffer(bufname);
         
         if (master == null)
         {
            throw new IllegalArgumentException("Invalid buffer name " + bufname);
         }
         
         return master;
      }
      
      /**
       * Combines any buffer scopes stored in the rollup list associated with
       * the given block into the global <code>active</code> buffer scope for
       * that block, then clears the rollup list.
       * <p>
       * This processing occurs in the case where an implicit expansion of
       * scope has occurred.  All buffers in the rollup list associated with
       * the block (that encloses the two buffers being combined) must be
       * merged with the scope being expanded. The scope being expanded is
       * found by obtaining the <code>active</code> buffer scope for this
       * buffer name.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same. Used to obtain the global 
       *           {@link BufferList} for this buffer.
       * @param    bbs
       *           The buffer-specific block state from which to obtain the
       *           rollup list.
       *
       * @throws   IllegalArgumentException
       *           If the buffer name is invalid.
       * @throws   RuntimeException
       *           If there is no <code>active</code> buffer scope for this
       *           buffer name.
       */
      public void rollupBuffers(String bufname, BufferBlockState bbs)
      {
         BufferList master = getMasterList(bufname);
         
         BufferScopeTracker active = master.getActive();
         
         if (active != null)
         {
            Iterator iter = bbs.rollupList.iterator();
            
            // walk through rollup list
            while (iter.hasNext())
            {
               BufferScopeTracker bst = (BufferScopeTracker) iter.next(); 
               
               // merge into the currently active buffer scope
               active.combine(bst);
            }
            
            // clear the rollup list
            bbs.rollupList.clear();               
            bbs.pendingRollupNode = null;
         }
         else
         {
            throw new RuntimeException("No active scope for " + bufname);
         }
      }
      
      /**
       * Moves any buffer scopes stored in the rollup list associated with
       * the given block into the global list of buffers, then clears the
       * rollup list.
       * <p>
       * This processing is done anytime it is known that the buffers in the
       * rollup list can no longer be merged with other buffer scopes (see
       * the {@link #rollupBuffers} method).
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same. Used to obtain the global 
       *           {@link BufferList} for this buffer.
       * @param    bbs
       *           The buffer-specific block state from which to obtain the
       *           rollup list.
       *
       * @throws   IllegalArgumentException
       *           If the buffer name is invalid.
       */
      public void finalizeBuffers(String bufname, BufferBlockState bbs)
      {
         BufferList master = getMasterList(bufname);
         
         Iterator iter = bbs.rollupList.iterator();
         
         // walk through rollup list
         while (iter.hasNext())
         {
            BufferScopeTracker bst = (BufferScopeTracker) iter.next(); 
            
            // add to the master list of all buffers
            master.addBufferScope(bst);
         }
         
         // clear the rollup list
         bbs.rollupList.clear();
      }
      
      /**
       * Helper to simplify construction of a {@link BufferScopeTracker} from
       * a rule set.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same. Used to obtain the global 
       *           {@link BufferList} for this buffer.
       * @param    ref
       *           The node which first references this buffer scope.
       * @param    stage
       *           <code>true</code> if the new buffer should be temporarily
       *           stored in the "staging" area of the {@link BufferList}
       *           such that the next block open will associate this buffer
       *           with the block.  This is used to associate strong and
       *           weak references with the block scope opened subsequently.
       *
       * @return   The newly constructed tracker containing a single
       *           reference and without any block scope assigned.
       *
       * @throws   IllegalArgumentException
       *           If the reference's buffer name is invalid.
       */
      public BufferScopeTracker createBuffer(String bufname, Aast ref, boolean stage)
      {
         Long               type = (Long) ref.getAnnotation("bufreftype");
         BufferScopeTracker bst  = new BufferScopeTracker(type.intValue()); 
         
         bst.addReference(ref);
         
         if (stage)
         {
            getMasterList(bufname).setStaged(bst);
         }
         
         return bst;
      }
      
      /**
       * Updates the state of the block and the global state for this buffer 
       * name with the resulting buffer scope that is the result of an
       * expansion.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same. Used to obtain the global 
       *           {@link BufferList} for this buffer.
       * @param    buf
       *           The expanded scope.
       *
       * @throws   IllegalArgumentException
       *           If the buffer name is invalid or if an active scope exists
       *           when this is called.
       */
      public void processExpansion(String bufname, BufferScopeTracker buf)
      {
         BufferList master = getMasterList(bufname);         
         
         if (master.getActive() != null)
         {
            throw new IllegalArgumentException("There should be no active " +
                                               "scope during an expansion.");
         }
         
         // set active
         master.setActive(buf);
         
         BufferBlockState bbs = getBlockState(bufname);
         
         // is the expansion is scoped to the current block?
         if (bbs.blockNode == buf.getScopeBlock())
         {
            // there may be a firewall expanded scope that is not in the
            // rollup list (because only weak scopes get inserted into that
            // list) and which is not the bindTarget
            if (bbs.lastExpanded != null           &&
                bbs.lastExpanded != bbs.bindTarget)
            {
               // in this case we need to combine is expanded scope with this
               // current expanded buf as part of the rollup process
               buf.combine(bbs.lastExpanded);
               
               // clear this so that it doesn't get added to the master list
               // separately in setExpanded() below
               bbs.lastExpanded = null;
            }
            
            // trigger a rollup
            rollupBuffers(bufname, bbs);
         }
         else
         {
            // set the pending rollup node
            bbs.pendingRollupNode = buf.getScopeBlock();
            
            // finalize the rollup list at this level
            finalizeBuffers(bufname, bbs);
         }
         
         // update block state
         setExpanded(master, bbs, buf);
      }
      
      /**
       * Updates the state of the containing block when a weak reference
       * is encountered.  Please note that this occurs BEFORE the block
       * associated with the weak reference actually opens, so the state
       * is set in the containing block.
       * <p>
       * This must only be called if there is no active scope and if there
       * is no pending free reference to which to bind (these cases are
       * expansions to be processed via {@link #processExpansion}, rather
       * than a standalone weak reference to be processed here).
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same. Used to obtain the global 
       *           {@link BufferList} for this buffer.
       * @param    buf
       *           The expanded scope.
       */
      public void processWeak(String bufname, BufferScopeTracker buf)
      {
         BufferBlockState bbs = getBlockState(bufname);
         
         // if this is the first weak reference in the current block, set
         // this as the first weak reference
         if (bbs.firstWeak == null)
         {
            bbs.firstWeak = buf;
         }
         
         // add to the rollup list
         bbs.rollupList.add(buf);
         
         // set the bind target
         bbs.bindTarget = buf;
      }
      
      /**
       * Create a new buffer scope and set it as a pending free reference (if
       * there is no bind target) OR force an expansion by merging this
       * reference with the current bind target.
       *
       * @param    bufname
       *           A name that uniquely represents all uses of this buffer
       *           name when it also references the specific given schema
       *           name. Since the same buffer names can be defined to 
       *           reference different schema entities between internal
       *           procedures, external procedures and functions, this
       *           name is needed to uniquely associate all references that
       *           are really the same. Used to obtain the global 
       *           {@link BufferList} for this buffer.
       * @param    ast
       *           The node representing the free reference.
       * @param    stage
       *           <code>true</code> to force a delayed assignment of the
       *           associated block until the next block open. This is
       *           needed to allow certain constructs (at this time, the
       *           FUNCTION -- KW_FUNCT -- PARAMETER -- KW_BUFFER -- record) 
       *           to emit in the proper location and with the proper block
       *           node.
       */
      public void processFree(String bufname, Aast ast, boolean stage)
      {
         BufferBlockState bbs = getBlockState(bufname);
         
         // is there a bind target?
         if (bbs.bindTarget == null)
         {
            // no bind target yet, create a new scope
            BufferScopeTracker buf = createBuffer(bufname, ast, stage);
            BufferList master = getMasterList(bufname);
            
            if (!stage)
            {
               String bufrefkey = (String) ast.getAnnotation("bufrefkey");
               if (bufrefkey != null)
               {
                  // get the definition AST
                  long astID = Long.parseLong(bufrefkey.substring(bufrefkey.lastIndexOf(',') + 1));
                  if (astID != -1)
                  {
                     Aast defAst = AstSymbolResolver.getAst(getResolver(), astID, true);
                     if (defAst != null)
                     {
                        if (defAst.getFilename().equals(ast.getFilename()))
                        {
                           // we are in the same file... check if we are in the same top-level block
                           Aast tl1 = ExpressionConversionWorker.findTopLevel(ast);
                           Aast tl2 = ExpressionConversionWorker.findTopLevel(defAst);
                           if (!tl1.getId().equals(tl2.getId()))
                           {
                              // the AST gets changed only if we are not in the same top-level block
                              ast = defAst;
                           }
                        }
                        else
                        {
                           // TODO: what should be done in this case?
                        }
                     }
                  }
               }
               // set the block (this will automatically search for the first
               // enclosing block node that has the record scoping property)
               buf.setScopeBlock(ast, master.getImplicit());
            }
            
            // save this as the pending free reference
            master.setPending(buf);
         }
         else
         {
            // there IS a bind target
            bbs.bindTarget.mergeReference(ast);
            
            // expand!
            processExpansion(bufname, bbs.bindTarget);
         }
      }
      
      /**
       * Simple helper to set the <code>lastExpanded</code>, the
       * <code>bindTarget</code> and <code>firstWeak</code> based on
       * a new expanded scope.
       *
       * @param    master
       *           The master list of buffers for the current buffer name
       *           being processed.
       * @param    state
       *           Block which is the target of the state update.
       * @param    buf
       *           The expanded scope.
       */
      private void setExpanded(BufferList         master,
                               BufferBlockState   state,
                               BufferScopeTracker buf)
      {
         // was there a previous different expanded scope?
         if (state.lastExpanded != null && state.lastExpanded != buf)
         {
            // this previous expanded scope can no longer ever be rolled up
            master.addBufferScope(state.lastExpanded);
         }
         
         state.lastExpanded = buf;
         state.bindTarget   = buf;
         state.firstWeak    = null;                           
      }
      
      /**
       * Helper method to match a partial buffer name (the portion after the
       * given delimiter) with the corresponding, partial name of any
       * registered buffer.  The first match (should be the only match) is
       * returned.
       *
       * @param   bufname
       *          Buffer name to be matched.
       * @param   delimiter
       *          Marker string within <code>bufname</code> (and presumably,
       *          within each registered buffer's name) which delimits the
       *          ignored, qualifying portion of the name, from the the
       *          significant portion of the name.
       *
       * @return  The buffer list associated with the first matching name, or
       *          <code>null</code> if no match is found.
       */
      private BufferList matchAfter(String bufname, String delimiter)
      {
         WorkArea wa = context.get();

         BufferList bl = null;
         
         int pos = bufname.indexOf(delimiter);
         String uname = (pos >= 0 ? bufname.substring(pos + 1) : bufname);
         Iterator iter = wa.allBufs.entrySet().iterator();
         while (iter.hasNext())
         {
            Map.Entry next = (Map.Entry) iter.next();
            String key = (String) next.getKey();
            pos = key.indexOf(delimiter);
            if (pos >= 0)
            {
               key = key.substring(pos + 1);
            }
            
            if (uname.equals(key))
            {
               bl = (BufferList) next.getValue();
               break;
            }
         }
         
         return bl;
      }
   }
   
   /**
    * Context local work area.
    */
   private static class WorkArea
   {
      /**
       * Stores each unique buffer name and maps it to the corresponding
       * {@link BufferList} instance. Provides a stable iteration order.
       */ 
      private Map<String, BufferList> allBufs = new LinkedHashMap<>();
      
      /**
       * Stores the block state for each block of a source file.  Each entry
       * in the stack is a <code>Map</code> containing one entry for each
       * possible buffer.  The entries are mapped with a key that is the
       * buffer name (a <code>String</code>) and the value is a
       * {@link BufferBlockState} instance.
       */
      private Stack<Map<String, BufferBlockState>> blocks = new Stack<>();
   }
}