FWD » Core Development » Conversion Tools

    dependsOn 'classes', 'copyResources', 'createCompileClasspathJar'

// Ahead-of-time generation of the expression classes of the shipped rule sets.

//

// A conversion otherwise generates, defines and JIT compiles a class for every expression of every

// rule set it loads, on every run.  Those rule sets are fixed content shipped with FWD, so the work

// is the same every time and is done here instead, once, and packaged into p2j.jar.

//

// Up-to-date checking is what keeps this off the critical path of an ordinary build:  the task

// reruns only when a rule file changes, or when the expression compiler which generates the classes

// changes.  The latter matters because a change to code generation alters what the correct bytecode

// is, and nothing at run time would detect a class generated by an older compiler.

// ============================================================================

// NOT yet wired into 'jar':  the generator currently produces an empty index, because compiling a

// rule set's expressions outside a running conversion does not resolve the pattern worker

// namespaces (see AotWarmer).  Wiring it in would add minutes to every build for no benefit.

tasks.register('generateAotExpressions', JavaExec) {

    description = 'Generate the expression classes of the shipped rule sets (experimental)'

    group = 'build'

    dependsOn 'classes', 'copyResources'

    def aotDir = layout.buildDirectory.dir('aot')

    def aotLevel = (project.findProperty('aotDebugLevel') ?: '0').toString()

    inputs.files(fileTree('rules'))

          .withPropertyName('ruleSets')

          .withPathSensitivity(PathSensitivity.RELATIVE)

    inputs.files(fileTree('src/com/goldencode/expr'))

          .withPropertyName('expressionCompiler')

          .withPathSensitivity(PathSensitivity.RELATIVE)

    // the level decides what the compiler emits, so it has to invalidate the output like any

    // other input;  without this, changing it would leave the previous level's classes in place

    inputs.property('debugLevel', aotLevel)

    outputs.dir(aotDir).withPropertyName('generatedClasses')

    // The generator loads every profile, and the pattern engine has plenty to say while doing so

    // -- none of it about this task.  Capture it all and surface only the one summary line, unless

    // the build is run with --info or the generator fails, in which case show everything.

    def captured = new ByteArrayOutputStream()

    standardOutput = captured

    errorOutput = captured

    ignoreExitValue = true

    mainClass = 'com.goldencode.p2j.pattern.AotWarmer'

    // compileJava writes to build/classes, and the dependency set is reached through the manifest

    // classpath jar the build already assembles for exactly this purpose

    classpath = files("${layout.buildDirectory.get()}/classes",

                      "${layout.buildDirectory.get()}/compile-classpath.jar")

                   .builtBy('classes', 'copyResources', 'createCompileClasspathJar')

    maxHeapSize = '4g'

    doFirst {

        // 0..3 only:  see DebugLevels.  A value outside that range silently falls through to no

        // runtime flags, producing level 0 classes labelled as something else, which is worse than

        // a failure.

        if (!(aotLevel ==~ /[0-3]/)) {

            throw new GradleException(

                "aotDebugLevel must be 0..3 (none, status, debug, trace), got '${aotLevel}'")

        }

        // The pattern engine expects a project home with a cfg/ directory.  Generation needs none

        // of what a real project configures -- no schema, no file set, no output root -- because

        // rule set variables carry explicit Java types and pattern worker libraries are plain Java

        // classes.  So a minimal home is synthesized here, pointing only at the rules to load.

        def aotHome = layout.buildDirectory.dir('aot-home').get().asFile

        def cfgDir = new File(aotHome, 'cfg')

        cfgDir.mkdirs()

        new File(cfgDir, 'p2j.cfg.xml').text = """<?xml version="1.0"?>

<!-- Generated by the generateAotExpressions task; not a project configuration. -->

<cfg>

   <global>

      <parameter name="P2J_RULES" value="${projectDir.path}/rules" />

      <parameter name="patpath"   value=".:\${P2J_RULES}/include:\${P2J_RULES}:" />

   </global>

   <profile name="default" default="true">

      <file-set/>

   </profile>

   <schema>

      <metadata name="standard"/>

   </schema>

</cfg>

"""

        workingDir = aotHome

        systemProperty 'P2J_HOME', aotHome.path

        // Must match the Dn debug level conversions run at (see the -SIdN argument in a project's

        // build.xml).  Override with -PaotDebugLevel=N.

        systemProperty 'p2j.aot.debug-level', aotLevel

        systemProperty 'p2j.aot.verbose', (project.findProperty('aotVerbose') ?: 'false')

        // every profile in the rules tree;  one which cannot be loaded is reported and skipped,

        // and its expressions simply keep being compiled at conversion time

        // Every profile in the rules tree except the call graph ones, which are a separate

        // pipeline and bring up their JanusGraph/BerkeleyDB store merely to be loaded.

        //

        // Resist pruning further by directory name.  The profiles under runtime/ are loaded by

        // the *server*, not by a conversion -- it converts dynamic queries through this same

        // pattern engine, see ConversionProfile in the persist package -- and dropping them on

        // the assumption that "runtime" meant something else silently cost the server its

        // generated classes.  progress/ and reports/ are likewise reached, from

        // ProgressTransformDriver and ReportDriver.  A profile which fails to load is reported

        // and skipped anyway, so guessing which ones matter buys nothing.

        def excluded = ['callgraph/']

        def profiles = fileTree('rules') { include '**/*.xml' }

            .files

            .collect { file(it).path - (projectDir.path + '/rules/') - '.xml' }

            .findAll { name -> !excluded.any { name.startsWith(it) } }

            .sort()

        logger.info("generateAotExpressions: ${profiles.size()} profiles")

        args = [aotDir.get().asFile.path] + profiles

    }

    doLast {

        def text = captured.toString()

        if (executionResult.get().exitValue != 0) {

            logger.error(text)

            throw new GradleException('generateAotExpressions failed')

        }

        if (logger.isInfoEnabled()) {

            logger.lifecycle(text.trim())

        }

        else {

            text.readLines()

                .findAll { it.startsWith('AOT expressions:') }

                .each { logger.lifecycle(it) }

        }

    }

}

// Identity of the inputs the generated expression classes were produced from.  The same value is

// written into both jars;  a conversion compares them and ignores the generated classes unless

// they agree, so a fwd-trpl.jar left behind from an older build is inert rather than wrong.

def aotStampValue = {

    def digest = java.security.MessageDigest.getInstance('SHA-256')

    def sources = []

    fileTree('rules').files.each { sources << it }

    fileTree('src/com/goldencode/expr').files.each { sources << it }

    sources.sort { it.path }.each { f ->

        digest.update((f.path - projectDir.path).getBytes('UTF-8'))

        digest.update(java.security.MessageDigest.getInstance('SHA-256').digest(f.bytes))

    }

    return digest.digest().encodeHex().toString().substring(0, 32)

}

tasks.register('aotStamp') {

    description = 'Compute the identity shared by p2j.jar and fwd-trpl.jar'

    // part of src/com/goldencode/expr is generated, so the generators have to have run before

    // its content can be hashed

    dependsOn 'classes'

    inputs.files(fileTree('rules')).withPropertyName('ruleSets')

    inputs.files(fileTree('src/com/goldencode/expr')).withPropertyName('expressionCompiler')

    outputs.dir(layout.buildDirectory.dir('aot-stamp')).withPropertyName('stamp')

    doLast {

        def value = aotStampValue()

        // the copy inside p2j.jar: what a conversion expects the generated classes to have been

        // built from

        def expected = layout.buildDirectory.file('aot-stamp/com/goldencode/expr/aot-expected.txt')

                             .get().asFile

        expected.parentFile.mkdirs()

        expected.text = value + '\n'

        logger.info("aotStamp: ${value}")

    }

}

// The generated expression classes, packaged apart from p2j.jar.  Separate because there are tens

// of thousands of them:  folding them into p2j.jar pushed it past the 64k zip entry limit, and it

// keeps an optional, regenerable artifact out of the one that has to be right.

tasks.register('trplJar', Jar) {

    dependsOn 'generateAotExpressions', 'aotStamp'

    archiveFileName = 'fwd-trpl.jar'

    destinationDirectory = file("${layout.buildDirectory.get()}/lib")

    from("${layout.buildDirectory.get()}/aot") {

        include 'com/goldencode/expr/aot/**'

    }

    doFirst {

        // the copy inside fwd-trpl.jar: what these classes were actually built from

        def stampFile = layout.buildDirectory.file('aot/com/goldencode/expr/aot/stamp.txt')

                              .get().asFile

        stampFile.parentFile.mkdirs()

        stampFile.text = aotStampValue() + '\n'

    }

}

// ============================================================================

              'createCompileClasspathJar', 'createTestsClasspathJar',

              'trplJar', 'aotStamp'

    // Only the expected stamp;  the generated classes themselves live in fwd-trpl.jar.

    from("${layout.buildDirectory.get()}/aot-stamp") {

        include 'com/goldencode/expr/aot-expected.txt'

    }

                // fwd-trpl.jar carries the expression classes generated from the shipped rule

                // sets.  Referenced here so that a project picks it up without touching its own

                // build, and optional at run time:  if it is absent or stale, expressions are

                // compiled exactly as they always were.

                'Class-Path': (['fwd-trpl.jar'] +

                               configurations.fwdAllRuntime.collect { it.name }).join(' ')

        // fwd-trpl.jar is needed here as well as for conversion:  the server converts dynamic

        // queries at runtime through the same pattern engine (see ConversionProfile).

        include 'p2j.jar', 'fwdaopltw.jar', 'fwd-trpl.jar', 'fwd_sheet.war', libname

/*

** Module   : AotGenerator.java

** Abstract : Build time generation of the expression classes of the shipped rule sets

**

** Copyright (c) 2026, Golden Code Development Corporation.

**

** -#- -I- --Date-- ----------------Description-----------------

** 001 TG  20260909 Initial version.

*/

package com.goldencode.expr;

import java.io.*;

import java.nio.charset.*;

import java.nio.file.*;

import java.security.*;

import java.util.*;

/**

 * Collects the expression classes of the shipped rule sets at build time, so that a conversion can

 * load them instead of generating them again on every run.

 *

 * <h3>How generation is driven</h3>

 *

 * This class does not walk the rule sets itself.  It switches the compiler into generation mode and

 * lets the normal machinery run:  a driver loads each profile and forces every expression in it to

 * compile, and each resulting class arrives here through {@link #emit}.  That keeps one code path

 * for producing a class, which is what makes a generated class trustworthy -- there is no second

 * implementation to drift from the first.

 *

 * <h3>Naming and sharing</h3>

 *

 * Classes are named from {@link Compiler#signature}, a description of the compilation which is

 * available before any code is assembled and which is verified to determine the emitted bytecode

 * exactly.  Occurrences whose compilations are interchangeable therefore land on one name and are

 * emitted once, which is why the class count is well below the occurrence count.

 * <p>

 * The names live in their own package, so they can never collide with the sequentially named

 * classes an ordinary compilation produces.

 *

 * @see AotIndex

 */

public final class AotGenerator

{

   /** Package holding every generated class. */

   static final String PACKAGE = AotIndex.PACKAGE;

   /** Flag indicating the compiler should name and emit classes for the index. */

   private static volatile boolean generating = false;

   /** Emitted classes, keyed by fully qualified name, so a shared class is written once. */

   private static final Map<String, byte[]> classes = new TreeMap<>();

   /** Index entries, keyed by expression occurrence name. */

   private static final Map<String, String> index = new TreeMap<>();

   /** Number of compilations which reached this collector. */

   private static int compiled = 0;

   /** Number of those which had no durable name and so could not be indexed. */

   private static int unnamed = 0;

   /** Number of class files actually written, after folding byte identical ones together. */

   private static int classesWritten = 0;

   /**

    * Report whether the compiler is running in generation mode.

    *

    * @return  <code>true</code> while a generation run is in progress.

    */

   static boolean isGenerating()

   {

      return generating;

   }

   /**

    * Turn generation mode on or off.

    *

    * @param   on

    *          <code>true</code> to start generating.

    */

   public static void setGenerating(boolean on)

   {

      generating = on;

   }

   /**

    * Compose the class name for a compilation, in the internal, slash separated form which

    * {@link ClassFile} requires and which is embedded in the generated constant pool.

    *

    * @param   signature

    *          The compilation's signature, from {@link Compiler#signature}.

    *

    * @return  Internal form of the class name.

    */

   static String internalName(String signature)

   {

      return (PACKAGE + "A" + AotIndex.digest(signature)).replace('.', '/');

   }

   /**

    * Accept a class the compiler has just assembled, together with the state a conversion needs in

    * order to bind it to a caller's scope without parsing the expression again.

    *

    * @param   expression

    *          Expression the class was generated for.

    * @param   classFile

    *          The assembled class.

    * @param   returnType

    *          Return type the blueprint must be configured with.

    * @param   libraries

    *          Library symbol to slot map.

    * @param   variables

    *          Variable symbol to slot map.

    */

   static synchronized void emit(Expression expression, ClassFile classFile, Class<?> returnType,

                                 Map<String, Integer> libraries, Map<String, Integer> variables)

   {

      String aotId = expression.getAotId();

      compiled++;

      if (aotId == null)

      {

         // not part of a rule set, so no conversion will ever look for it

         unnamed++;

         return;

      }

      String className = classFile.getFullyQualifiedName();

      try

      {

         if (!classes.containsKey(className))

         {

            ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);

            DataOutputStream dos = new DataOutputStream(baos);

            classFile.write(dos);

            dos.flush();

            classes.put(className, baos.toByteArray());

         }

         index.put(aotId,

                   className.substring(PACKAGE.length()) + '|' +

                   expression.getRuntimeFlags() + '|' +

                   AotIndex.digest(expression.getInfix()) + '|' +

                   ((returnType == null) ? "null" : returnType.getName()) + '|' +

                   String.join(",", slotNames(libraries)) + '|' +

                   String.join(",", slotNames(variables)));

      }

      catch (IOException exc)

      {

         throw new IllegalStateException("could not serialize " + className, exc);

      }

   }

   /**

    * Write everything collected so far:  one class file per generated class, laid out by package so

    * that the build can fold them straight into the artifact, plus the index which maps occurrences

    * onto them.

    *

    * @param   outputDir

    *          Root directory to write into.

    * @param   debugLevel

    *          Debug level these classes were generated for.  Recorded so that a conversion running

    *          at a different level can say so precisely instead of just finding nothing.

    *

    * @throws  IOException

    *          if writing fails.

    */

   public static synchronized void write(File outputDir, int debugLevel)

   throws IOException

   {

      File classDir = new File(outputDir, PACKAGE.replace('.', '/'));

      classDir.mkdirs();

      // Names come from the compilation signature, which is deliberately over-inclusive:  it

      // separates compilations which would in fact assemble to identical bytecode.  Fold those

      // together now, by fingerprinting each class with its own name masked out, so that the

      // artifact carries one copy and a conversion loads and verifies one class instead of

      // several.  Class loading is the dominant cost of using these at run time, so this is worth

      // more than the disk space it saves.

      Map<String, String> canonical = new HashMap<>(classes.size());

      Map<String, String> alias = new HashMap<>(classes.size());

      for (Map.Entry<String, byte[]> next : classes.entrySet())

      {

         String name = next.getKey();

         String print = fingerprint(next.getValue(), name.replace('.', '/'));

         String first = canonical.putIfAbsent(print, name);

         alias.put(name, (first == null) ? name : first);

      }

      int written = 0;

      for (Map.Entry<String, byte[]> next : classes.entrySet())

      {

         String name = next.getKey();

         if (!name.equals(alias.get(name)))

         {

            // a duplicate of one already being written

            continue;

         }

         Files.write(new File(classDir, name.substring(PACKAGE.length()) + ".class").toPath(),

                     next.getValue());

         written++;

      }

      // point every occurrence at the surviving copy of its class

      for (Map.Entry<String, String> next : index.entrySet())

      {

         String value = next.getValue();

         int bar = value.indexOf('|');

         String canon = alias.get(PACKAGE + value.substring(0, bar));

         if (canon != null)

         {

            next.setValue(canon.substring(PACKAGE.length()) + value.substring(bar));

         }

      }

      // One index per profile.  A conversion loads profiles one at a time, so this keeps both the

      // parse cost and the resident map proportional to the profile in hand rather than to the

      // whole rule base -- which is nearly half a million occurrences.

      File indexDir = new File(classDir, "index");

      indexDir.mkdirs();

      Map<String, List<String>> byProfile = new TreeMap<>();

      for (Map.Entry<String, String> next : index.entrySet())

      {

         String aotId = next.getKey();

         int hash = aotId.lastIndexOf('#');

         String profile = aotId.substring(0, hash);

         byProfile.computeIfAbsent(profile, k -> new ArrayList<>())

                  .add(aotId.substring(hash + 1) + '|' + next.getValue());

      }

      for (Map.Entry<String, List<String>> next : byProfile.entrySet())

      {

         File target = new File(indexDir, AotIndex.indexFileName(next.getKey()));

         try (PrintWriter out = new PrintWriter(

                 new OutputStreamWriter(

                    new BufferedOutputStream(new FileOutputStream(target)),

                    StandardCharsets.UTF_8)))

         {

            out.println(AotIndex.MAGIC);

            for (String line : next.getValue())

            {

               out.println(line);

            }

         }

      }

      // what these classes were generated for, so a mismatch can be reported precisely

      Files.write(new File(classDir, "meta.txt").toPath(),

                  ("debug-level=" + debugLevel + "\n").getBytes(StandardCharsets.UTF_8));

      classesWritten = written;

   }

   /**

    * Report how many occurrences have been indexed.

    *

    * @return  Occurrence count.

    */

   public static synchronized int occurrences()

   {

      return index.size();

   }

   /**

    * Report how many distinct classes have been generated.

    *

    * @return  Class count.

    */

   public static synchronized int classCount()

   {

      return (classesWritten == 0) ? classes.size() : classesWritten;

   }

   /**

    * Fingerprint class file bytes, ignoring every occurrence of the class's own internal name.

    * <p>

    * That name is the one part which differs between two otherwise identical classes, and it is a

    * constant pool UTF8 entry preceded by its own two byte length, so both are dropped.

    *

    * @param   raw

    *          Class file bytes.

    * @param   internalName

    *          The class's own name in internal form.

    *

    * @return  Hex encoded digest of the name independent content.

    */

   private static String fingerprint(byte[] raw, String internalName)

   {

      try

      {

         byte[] needle = internalName.getBytes(StandardCharsets.UTF_8);

         MessageDigest digest = MessageDigest.getInstance("SHA-256");

         int pos = 0;

         for (int i = 0; i <= raw.length - needle.length; )

         {

            if (matches(raw, i, needle))

            {

               digest.update(raw, pos, Math.max(pos, i - 2) - pos);

               i += needle.length;

               pos = i;

            }

            else

            {

               i++;

            }

         }

         digest.update(raw, pos, raw.length - pos);

         StringBuilder buf = new StringBuilder(64);

         for (byte b : digest.digest())

         {

            buf.append(Character.forDigit((b >> 4) & 0xf, 16));

            buf.append(Character.forDigit(b & 0xf, 16));

         }

         return buf.toString();

      }

      catch (NoSuchAlgorithmException exc)

      {

         throw new IllegalStateException(exc);

      }

   }

   /**

    * Test whether <code>needle</code> occurs in <code>raw</code> at <code>at</code>.

    *

    * @param   raw

    *          Buffer to search.

    * @param   at

    *          Offset at which to compare.

    * @param   needle

    *          Bytes to look for.

    *

    * @return  <code>true</code> on a match.

    */

   private static boolean matches(byte[] raw, int at, byte[] needle)

   {

      for (int i = 0; i < needle.length; i++)

      {

         if (raw[at + i] != needle[i])

         {

            return false;

         }

      }

      return true;

   }

   /**

    * Render a symbol to slot map as a list ordered by slot.

    *

    * @param   slots

    *          Map of symbol name to slot index.

    *

    * @return  Symbol names in slot order.

    */

   private static List<String> slotNames(Map<String, Integer> slots)

   {

      String[] byIndex = new String[slots.size()];

      for (Map.Entry<String, Integer> next : slots.entrySet())

      {

         byIndex[next.getValue()] = next.getKey();

      }

      return Arrays.asList(byIndex);

   }

   /**

    * Not instantiable.

    */

   private AotGenerator()

   {

   }

}

/*

** Module   : AotIndex.java

** Abstract : Index of expression classes generated when FWD itself was built

**

** Copyright (c) 2026, Golden Code Development Corporation.

**

** -#- -I- --Date-- ----------------Description-----------------

** 001 TG  20260909 Initial version.

*/

package com.goldencode.expr;

import java.io.*;

import java.nio.charset.*;

import java.security.*;

import java.util.*;

/**

 * Index of the expression classes which were generated when FWD was built, and which ship inside

 * the same artifact as the rule sets they came from.

 *

 * <h3>What this replaces</h3>

 *

 * Left to itself a conversion generates, defines and JIT compiles a class for every expression of

 * every rule set it loads, on every run.  Those rule sets are fixed content shipped with FWD, so

 * the work is the same every time and can be done once, at build time.  A run which finds an

 * occurrence here loads an ordinary class from the classpath instead.

 *

 * <h3>How an occurrence is named</h3>

 *

 * Every expression belonging to a rule set is created during a profile load, which reads a fixed

 * set of files in document order on a single thread, so each occurrence has a stable position in

 * that sequence.  {@link Expression#assignAotId} turns that into a durable name, and this index is

 * keyed on it.  Because the index and the rule sets are produced by one build and shipped in one

 * artifact, they cannot disagree about that sequence.

 *

 * <h3>What an entry holds beyond the class</h3>

 *

 * The class alone is not enough:  {@link Compiler#instantiate} binds a compiled expression to the

 * libraries and variables of the calling scope through the symbol-to-slot maps that parsing

 * produced, and the blueprint needs its return type.  Recording those is what removes parsing from

 * the fast path.

 *

 * <h3>Failing safe</h3>

 *

 * Every entry carries a digest of the expression text it was generated from, checked before the

 * class is used.  In a released build that check can never fail;  it exists for the developer case

 * where rules have been edited without regenerating, and it turns what would be a wrong class into

 * an ordinary compilation.  A missing index, an unreadable entry, an absent class -- all degrade to

 * compiling, never to misbehaving.

 */

final class AotIndex

{

   /** Package holding every generated class. */

   static final String PACKAGE = "com.goldencode.expr.aot.";

   /** Resource directory holding the per profile indexes, relative to the classpath root. */

   static final String RESOURCE_DIR = "com/goldencode/expr/aot/index/";

   /** Identity of the inputs the generated classes were built from, inside fwd-trpl.jar. */

   static final String STAMP_RESOURCE = "com/goldencode/expr/aot/stamp.txt";

   /** Identity those classes are expected to have, inside p2j.jar. */

   static final String EXPECTED_RESOURCE = "com/goldencode/expr/aot-expected.txt";

   /** What the generated classes were built for, inside fwd-trpl.jar. */

   static final String META_RESOURCE = "com/goldencode/expr/aot/meta.txt";

   /** Format marker of the index. */

   static final String MAGIC = "#fwd-aot 2";

   /** Occurrences served from the index. */

   private static int hits = 0;

   /** Occurrences which had to be compiled. */

   private static int misses = 0;

   /** Occurrences compiled because they belong to no profile -- expression text built at run time. */

   private static int unnamedMisses = 0;

   /** Hits and misses per profile, so a shortfall can be attributed rather than guessed at. */

   private static final Map<String, int[]> byProfile = new TreeMap<>();

   /** Entries of the profile currently loaded, keyed by occurrence ordinal. */

   private static volatile Map<Integer, Entry> entries = Collections.emptyMap();

   /** Profile whose entries are resident, or <code>null</code>. */

   private static volatile String resident = null;

   /**

    * Flag disabling use of the generated classes, so a conversion compiles expressions exactly as

    * it would without them.  Set <code>P2J_AOT_DISABLE</code> in the environment.  Kept as an

    * escape hatch:  if a generated class is ever suspected, this turns the mechanism off without

    * rebuilding or repackaging anything.

    */

   private static final boolean DISABLED = (System.getenv("P2J_AOT_DISABLE") != null);

   /**

    * The artifact holding the generated classes -- fwd-trpl.jar -- or <code>null</code> when it is

    * absent, unreadable, or does not match this build.

    * <p>

    * It is located once, by asking the classpath for its stamp, and thereafter read directly.  The

    * conversion classpath is several hundred jars, so searching it per class is what this avoids.

    */

   private static final Object SOURCE = openSource();

   /** Loader which resolves the generated classes straight out of {@link #SOURCE}. */

   private static final AotClassLoader LOADER = new AotClassLoader(AotIndex.class.getClassLoader());

   /**

    * Not instantiable.

    */

   private AotIndex()

   {

   }

   /**

    * Compose the resource name holding a profile's index.  Profile names contain path separators,

    * which are folded so that every index is one flat resource.

    *

    * @param   profile

    *          Profile name.

    *

    * @return  Resource file name.

    */

   public static String indexFileName(String profile)

   {

      return profile.replace('/', '.') + ".txt";

   }

   static

   {

      Runtime.getRuntime().addShutdownHook(new Thread(AotIndex::report));

   }

   /**

    * Describe what the generated classes were built for.

    *

    * @return  A short description, for reporting.

    */

   private static String describeGeneratedFor()

   {

      String meta = text(bytesOf(META_RESOURCE));

      return (meta == null) ? "an unrecorded debug level" : meta.replace('=', ' ');

   }

   /**

    * Report whether any entry is currently resident.

    *

    * @return  <code>true</code> if a lookup could succeed.

    */

   static boolean isAvailable()

   {

      return !entries.isEmpty();

   }

   /**

    * Make the generated entries of a profile resident, replacing whichever profile was resident

    * before.  Called as a profile's rule sets begin loading.

    *

    * @param   profile

    *          Profile being loaded, or <code>null</code> to drop what is resident.

    */

   static synchronized void selectProfile(String profile)

   {

      if (DISABLED || profile == null || profile.equals(resident))

      {

         return;

      }

      resident = profile;

      entries = read(profile);

   }

   /**

    * Load one of the generated classes.

    *

    * @param   className

    *          Fully qualified name of the class.

    *

    * @return  The class.

    *

    * @throws  ClassNotFoundException

    *          if it is not present.

    */

   static Class<?> loadClass(String className)

   throws ClassNotFoundException

   {

      return LOADER.loadClass(className, true);

   }

   /**

    * Note that an occurrence was served from the index.

    */

   static synchronized void hit()

   {

      hits++;

      tally(resident, 0);

   }

   /**

    * Note that an occurrence had to be compiled.

    */

   static synchronized void miss()

   {

      misses++;

      tally(resident, 1);

   }

   /**

    * Note that an occurrence had to be compiled because it has no durable name at all, which means

    * its text was assembled at run time rather than read from a rule set.  Nothing can be

    * generated for those.

    */

   static synchronized void unnamedMiss()

   {

      misses++;

      unnamedMisses++;

   }

   /**

    * Record an outcome against the profile currently loaded.

    *

    * @param   profile

    *          Profile in force, or <code>null</code>.

    * @param   slot

    *          0 for a hit, 1 for a miss.

    */

   private static void tally(String profile, int slot)

   {

      byProfile.computeIfAbsent((profile == null) ? "<none>" : profile, k -> new int[2])[slot]++;

   }

   /**

    * Look up the class generated for an expression occurrence.

    *

    * @param   aotId

    *          Durable name of the occurrence, or <code>null</code> if it has none.

    * @param   infix

    *          The occurrence's expression text, checked against the text the entry was generated

    *          from.

    * @param   flags

    *          The occurrence's runtime flags, likewise checked.

    *

    * @return  The entry, or <code>null</code> if there is none or it does not describe this

    *          expression, in which case the caller compiles as usual.

    */

   static Entry lookup(String aotId, String infix, int flags)

   {

      if (aotId == null || entries.isEmpty())

      {

         return null;

      }

      int hash = aotId.lastIndexOf('#');

      if (hash < 0 || !aotId.regionMatches(0, resident, 0, hash))

      {

         // an occurrence of some other profile;  only one profile's entries are resident

         return null;

      }

      Entry entry = entries.get(Integer.valueOf(aotId.substring(hash + 1)));

      if (entry == null || entry.flags != flags || !entry.infixDigest.equals(digest(infix)))

      {

         return null;

      }

      return entry;

   }

   /**

    * Digest an expression text.

    *

    * @param   text

    *          Text to digest.

    *

    * @return  Truncated hex encoded digest.

    */

   static String digest(String text)

   {

      try

      {

         byte[] raw = MessageDigest.getInstance("SHA-256")

                                   .digest(text.getBytes(StandardCharsets.UTF_8));

         StringBuilder buf = new StringBuilder(32);

         for (byte b : raw)

         {

            buf.append(Character.forDigit((b >> 4) & 0xf, 16));

            buf.append(Character.forDigit(b & 0xf, 16));

         }

         return buf.substring(0, 24);

      }

      catch (NoSuchAlgorithmException exc)

      {

         throw new IllegalStateException(exc);

      }

   }

   /**

    * Read the index from the classpath.

    *

    * @return  The entries, or an empty map if no index is present or it cannot be parsed.

    */

   private static Map<Integer, Entry> read(String profile)

   {

      byte[] raw = bytesOf(RESOURCE_DIR + indexFileName(profile));

      if (raw == null)

      {

         // FWD built without generated expressions for this profile;  it compiles as it always did

         return Collections.emptyMap();

      }

      Map<Integer, Entry> loaded = new HashMap<>(16384);

      try (BufferedReader reader = new BufferedReader(

              new InputStreamReader(new ByteArrayInputStream(raw), StandardCharsets.UTF_8)))

      {

         String header = reader.readLine();

         if (header == null || !header.startsWith(MAGIC))

         {

            System.err.println("AotIndex: unrecognized index format, ignoring");

            return Collections.emptyMap();

         }

         String line;

         while ((line = reader.readLine()) != null)

         {

            if (line.isEmpty() || line.charAt(0) == '#')

            {

               continue;

            }

            String[] f = line.split("\\|", 7);

            if (f.length == 7)

            {

               loaded.put(Integer.valueOf(f[0]),

                          new Entry(PACKAGE + f[1], Integer.parseInt(f[2]), f[3], f[4],

                                    split(f[5]), split(f[6])));

            }

         }

      }

      catch (Exception exc)

      {

         System.err.println("AotIndex: could not read index, ignoring: " + exc);

         return Collections.emptyMap();

      }

      return loaded;

   }

   /**

    * Split a comma separated symbol list.

    *

    * @param   text

    *          Encoded list, possibly empty.

    *

    * @return  The symbol names, in slot order.

    */

   private static List<String> split(String text)

   {

      return text.isEmpty() ? Collections.emptyList() : Arrays.asList(text.split(","));

   }

   /**

    * Locate fwd-trpl.jar, but only if it was generated from the same inputs as this p2j.jar.

    * <p>

    * The two carry a shared stamp over the rule sets and the expression compiler.  Disagreement

    * means the generated classes describe rules or code generation which no longer exist, so they

    * are ignored outright and every expression is compiled as it always was.  That is the whole

    * staleness story:  the artifact is optional and regenerable, and being wrong is never a

    * possible outcome of it being old.

    *

    * @return  An open {@link java.util.jar.JarFile}, a {@link File} directory, or

    *          <code>null</code> if the generated classes should not be used.

    */

   private static Object openSource()

   {

      try

      {

         ClassLoader loader = AotIndex.class.getClassLoader();

         String expected = text(loader.getResourceAsStream(EXPECTED_RESOURCE));

         if (expected == null)

         {

            // this FWD build ships no generated classes

            return null;

         }

         java.net.URL url = loader.getResource(STAMP_RESOURCE);

         if (url == null)

         {

            // fwd-trpl.jar is not on the classpath;  perfectly fine, just slower

            return null;

         }

         String actual = text(url.openStream());

         if (!expected.equals(actual))

         {

            System.err.println("AotIndex: fwd-trpl.jar was generated from different sources than " +

                                  "this p2j.jar (expected " + expected + ", found " + actual +

                                  "); ignoring it and compiling expressions normally");

            return null;

         }

         if ("jar".equals(url.getProtocol()))

         {

            java.net.JarURLConnection conn = (java.net.JarURLConnection) url.openConnection();

            conn.setUseCaches(false);

            return new java.util.jar.JarFile(new File(conn.getJarFileURL().toURI()));

         }

         if ("file".equals(url.getProtocol()))

         {

            // exploded on disk:  climb back out of the resource path to the classpath root

            File root = new File(url.toURI());

            for (int i = 0; i < STAMP_RESOURCE.split("/").length; i++)

            {

               root = root.getParentFile();

            }

            return root;

         }

         return null;

      }

      catch (Exception exc)

      {

         System.err.println("AotIndex: could not open fwd-trpl.jar, compiling normally: " + exc);

         return null;

      }

   }

   /**

    * Read a small text resource, trimmed.

    *

    * @param   in

    *          Stream to read, possibly <code>null</code>.  Closed on return.

    *

    * @return  The trimmed content, or <code>null</code>.

    */

   private static String text(InputStream in)

   {

      if (in == null)

      {

         return null;

      }

      try (InputStream stream = in)

      {

         return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();

      }

      catch (IOException exc)

      {

         return null;

      }

   }

   /**

    * Interpret bytes as trimmed text.

    *

    * @param   raw

    *          Bytes, possibly <code>null</code>.

    *

    * @return  The trimmed content, or <code>null</code>.

    */

   private static String text(byte[] raw)

   {

      return (raw == null) ? null : new String(raw, StandardCharsets.UTF_8).trim();

   }

   /**

    * Read one entry from the artifact.

    *

    * @param   path

    *          Entry path, relative to the artifact root.

    *

    * @return  The bytes, or <code>null</code> if absent.

    */

   private static byte[] bytesOf(String path)

   {

      try

      {

         if (SOURCE instanceof java.util.jar.JarFile)

         {

            java.util.jar.JarFile jar = (java.util.jar.JarFile) SOURCE;

            java.util.zip.ZipEntry entry = jar.getEntry(path);

            if (entry == null)

            {

               return null;

            }

            try (InputStream in = jar.getInputStream(entry))

            {

               return in.readAllBytes();

            }

         }

         if (SOURCE instanceof File)

         {

            File file = new File((File) SOURCE, path);

            return file.isFile() ? java.nio.file.Files.readAllBytes(file.toPath()) : null;

         }

         // no usable artifact:  nothing to read, and the caller compiles instead

         return null;

      }

      catch (IOException exc)

      {

         return null;

      }

   }

   /**

    * Report what the generated classes did for this run.  A run which loaded an index but matched

    * nothing is called out:  that is what a conversion looks like when its debug level differs

    * from the one the classes were generated for, and it would otherwise be invisible.

    */

   private static void report()

   {

      if (DISABLED)

      {

         return;

      }

      System.err.println("AotIndex: served " + hits + ", compiled " + misses +

                         ((unnamedMisses == 0) ? ""

                                               : (" (" + unnamedMisses +

                                                  " of them expressions built at run time, which " +

                                                  "cannot be generated ahead of time)")));

      // Attribute the shortfall.  A profile which misses wholesale was not generated -- excluded

      // from the build, or it failed to load in the generator -- and that is worth knowing,

      // because it is fixable in a way that individual misses are not.

      byProfile.entrySet()

               .stream()

               .filter(e -> e.getValue()[1] > 0)

               .sorted((a, b) -> b.getValue()[1] - a.getValue()[1])

               .limit(10)

               .forEach(e -> System.err.println("AotIndex:    " + e.getKey() + " served " +

                                                e.getValue()[0] + ", compiled " +

                                                e.getValue()[1]));

      if (hits == 0 && misses > 0)

      {

         System.err.println("AotIndex: WARNING - an index was found but no occurrence matched it. " +

                               "The usual cause is a conversion debug level (the Dn option) different " +

                               "from the one the classes were generated for; expressions are being " +

                               "compiled as if the generated classes were absent.");

      }

   }

   /**

    * One indexed occurrence:  which class serves it, and the state needed to bind that class to a

    * caller's scope without parsing the expression again.

    */

   static final class Entry

   {

      /** Fully qualified name of the class which serves this occurrence. */

      final String className;

      /** Runtime flags the class was generated under. */

      final int flags;

      /** Digest of the expression text the class was generated from. */

      final String infixDigest;

      /** Name of the blueprint's return type. */

      final String returnType;

      /** Library symbol names, in slot order. */

      final List<String> libraries;

      /** Variable symbol names, in slot order. */

      final List<String> variables;

      /**

       * Constructor.

       *

       * @param   className

       *          Fully qualified name of the class which serves this occurrence.

       * @param   flags

       *          Runtime flags the class was generated under.

       * @param   infixDigest

       *          Digest of the expression text the class was generated from.

       * @param   returnType

       *          Name of the blueprint's return type.

       * @param   libraries

       *          Library symbol names, in slot order.

       * @param   variables

       *          Variable symbol names, in slot order.

       */

      Entry(String className, int flags, String infixDigest, String returnType,

            List<String> libraries, List<String> variables)

      {

         this.className   = className;

         this.flags       = flags;

         this.infixDigest = infixDigest;

         this.returnType  = returnType;

         this.libraries   = libraries;

         this.variables   = variables;

      }

   }

   /**

    * Loader for the generated expression classes.

    *

    * <h3>Why not the application loader</h3>

    *

    * The conversion classpath is several hundred jars.  Resolving a generated class through the

    * application loader searches all of them, and doing that once per generated class costs

    * seconds.  This loader knows the single artifact they live in, so a load is one zip entry read.

    *

    * <h3>Why loadClass is overridden and not just findClass</h3>

    *

    * The inherited <code>loadClass</code> is parent first:  it would consult the application loader

    * -- and therefore search that whole classpath -- before ever calling {@link #findClass}, which

    * is exactly the cost this loader exists to avoid.  Generated classes are never on the parent's

    * classpath, so for them the order is inverted.  Everything else, including the superclass of

    * every generated class and every type its code refers to, delegates normally.

    */

   private static final class AotClassLoader extends ClassLoader

   {

      static

      {

         // per class name locking rather than one lock for the loader:  conversion threads load

         // these concurrently

         registerAsParallelCapable();

      }

      /**

       * Constructor.

       *

       * @param   parent

       *          Loader to delegate everything but the generated classes to.

       */

      AotClassLoader(ClassLoader parent)

      {

         super(parent);

      }

      /**

       * {@inheritDoc}

       */

      @Override

      protected Class<?> loadClass(String name, boolean resolve)

      throws ClassNotFoundException

      {

         if (!name.startsWith(PACKAGE))

         {

            return super.loadClass(name, resolve);

         }

         synchronized (getClassLoadingLock(name))

         {

            Class<?> found = findLoadedClass(name);

            if (found == null)

            {

               found = findClass(name);

            }

            if (resolve)

            {

               resolveClass(found);

            }

            return found;

         }

      }

      /**

       * {@inheritDoc}

       */

      @Override

      protected Class<?> findClass(String name)

      throws ClassNotFoundException

      {

         byte[] data = bytesOf(name.replace('.', '/') + ".class");

         if (data == null)

         {

            throw new ClassNotFoundException(name);

         }

         return defineClass(name, data, 0, data.length);

      }

   }

}

** 019 TG  20260909          compile() takes the compile-time scope, which is now part of the

**                           compiler cache key.

import java.util.concurrent.*;

   /**

    * Compilers restored from the classes generated when FWD was built, keyed by the whole of the

    * index entry they were built from -- class, return type and both slot maps.  See

    * {@link #restore} for why the class name alone is not a sufficient key.

    */

   private static final Map<String, Compiler> RESTORED = new ConcurrentHashMap<>();

    * @param   scope

    *          Scope in force for this compilation.  It must be the scope the caller resolved

    *          before this call, not one re-derived here, so that the lookup and the publication

    *          agree on the key.

   static CompiledExpression compile(Expression expression, Scope scope)

      // Check if there is already a compiler for this expression in this scope.  The lookup is

      // scope aware because a generated class is specific to the scope which compiled it:  the

      // types of the variables it reads are frozen into its bytecode.

      Compiler compiler = expression.getCompiler(scope);

      expression.putCompiler(compiler, scope);

         // Everything the generated code depends on is resolved at this point and nothing has been

         // assembled yet, which is both where the compilation can be described and therefore where

         // a name derived from that description can be chosen.  Only the generator needs such a

         // name;  an ordinary run keeps the sequential naming, and the two live in different

         // packages so they can never collide.

         String internal;

         if (AotGenerator.isGenerating())

         {

            internal = AotGenerator.internalName(signature());

         }

         else

         {

            internal = PKG_PREFIX + CLASS_NAME_PREFIX + classCount.getAndIncrement();

         }

         // Create a class file object into which to install execute method.  Note that ClassFile

         // takes the internal, slash separated form, which is what ends up in the constant pool.

         classFile = new ClassFile(internal, CLASS_COMPILEDEXPRESSION);

         // Get constant pool, which is needed during assembly and

         // compilation.

         constantPool = classFile.getConstantPool();