FWD » Core Development » Conversion Tools

To translate the above configuration requirements into a sample p2j.cfg.xml, I understand it should look something like this:

<?xml version="1.0"?>

<!-- P2J main configuration -->
<cfg>

   <global>

      <!-- default conversion tools "internal" values -->
      <parameter name="P2J_RULES"        value="${P2J_HOME}/p2j/rules" />
      <parameter name="patpath"          value=".:${P2J_HOME}/pattern:${P2J_RULES}/include:${P2J_RULES}:" />
      <parameter name="registry"         value="./cfg/registry.xml" />
      <parameter name="rootlist"         value="./cfg/default_rootlist.xml" />    
      <parameter name="matchlist"        value="./cfg/default_matchlist.xml" />
      <parameter name="datanames"        value="./cfg/default_datanames.xml" />

      <!-- default values from the original system on which Progress 4GL ran -->
      <parameter name="propath"          value="${P2J_HOME}:${P2J_HOME}/abl/default/path/1:${P2J_HOME}/abl/default/path/2:${P2J_HOME}/abl/default/path/3:" />
      <parameter name="basepath"         value="./abl" />
      <parameter name="include-spec"     value="*.[fhiv]" />
      <parameter name="oo-skeleton-path" value="./skeleton" />

      <!-- conversion output -->
      <parameter name="output-root"      value="./src" />
      <parameter name="pkgroot"          value="com.acme" />
      <parameter name="merge-point"      value="/server/default/runtime/default/" />
      <parameter name="comments"         value="true" />
      <parameter name="foreign-keys"     value="false" />
      <parameter name="opsys"            value="UNIX" />

      <!-- default file-set -->
      <file-set>
         <directory recursive="true" path="./abl/" spec="(*.[pPwWtT]|*.cls)" />
         <exclude filter="./abl/possenet/*" />
      </file-set>

   </global>

   <schema>

      <namespace name="acme" default="true" />
      <namespace name="standard" default="true" />

      <metadata name="standard">
         <table name="_area" />
         <table name="_db" />
         <table name="_file" />
         <table name="_file-trig" />
         <table name="_field" />
         <table name="_field-trig" />
         <table name="_index" />
         <table name="_index-field" />
         <table name="_user" />
         <table name="_connect" />
         <table name="_myconnection" />
         <table name="_database-feature" />
         <table name="_filelist" />
         <table name="_startup" />
         <table name="_lock" />
         <table name="_usertablestat" />
         <table name="_sequence" />
      </metadata>
   </schema>

   <profile name="module1">

      <file-set>
         <directory recursive="true" path="./abl/common/" spec="(*.[pPwWtT]|*.cls)" />
         <directory recursive="true" path="./abl/module1/oo/" spec="*.cls" />
         <directory recursive="true" path="./abl/module1/legacy/" spec="*.[pPwW]" />
         <directory recursive="true" path="./abl/module1/trig/" spec="*.[tT]" />
         <exclude filter="./abl/module1/legacy/broken/*" />
      </file-set>

      <parameter name="merge-point"      value="/server/default/runtime/module1/" />
      <parameter name="propath"          value="${P2J_HOME}:${P2J_HOME}/abl/common/:${P2J_HOME}/abl/module1:${P2J_HOME}/abl/some/other/path:" />

      <schema>
         <namespace name="module1" default="true" />
         <namespace name="standard" />
      </schema>

   </profile>

   <profile name="module2">

      <file-set>
         <directory recursive="true" path="./abl/common/" spec="(*.[pPwWtT]|*.cls)" />
         <directory recursive="true" path="./abl/module2/main/" spec="(*.[pPwWtT]|*.cls)" />
      </file-set>

      <parameter name="datanames"        value="./some/other/location/module2_datanames.xml" />
      <parameter name="propath"          value="${P2J_HOME}:${P2J_HOME}/abl/common/:${P2J_HOME}/abl/module2:${P2J_HOME}/abl/module2/main:" />

      <schema>
         <namespace name="module2" default="true" />
         <namespace name="standard" />
      </schema>

   </profile>

   <profile name="module2-test">

      <file-set>
         <directory recursive="true" path="./abl/common/" spec="(*.[pPwWtT]|*.cls)" />
         <directory recursive="true" path="./abl/module2/main/" spec="(*.[pPwWtT]|*.cls)" />
         <directory recursive="true" path="./abl/module2/test/" spec="*.[pP]" />
      </file-set>

      <parameter name="propath"          value="${P2J_HOME}:${P2J_HOME}/abl/common/:${P2J_HOME}/abl/module2/main:${P2J_HOME}/abl/module2/test:" />

      <schema>
         <namespace name="module2_unittest" default="true" />
         <namespace name="standard" />
      </schema>

   </profile>

</cfg>

Is this the right vision?

It seems like some of the global parameters don't make sense to override (e.g., registry, oo-skeleton-path, etc.).

I'm assuming (for now) that profile *.df files will be found in ./data/<profile-name>/.

At this moment the structure of the configuration file is:

<cfg>
  <global>...</global>

  <schema>
    <!-- unnamed profile -->
    <namespace name="ns_0.1"/>
    <namespace name="ns_0.2"/>
    <!-- end of unnamed profile -->

    <profile name="p1">
       <namespace name="ns_1.1"/>
       <namespace name="ns_1.2"/>
    </profile>
    ...
    <profile  name="pk">
       <namespace name="ns_k.1"/>
       <namespace name="ns_k.2"/>
    </profile>

    <metadata name="standard" />
  </schema>
</cfg>

• a set of property-es specific to respective profile,
• the schema sub-node with a set of namespaces and
• the set of files to be processed.

The metadata should be still common and defined only once.

Is this the right vision?

Mostly, yes. The part I don't understand is why there is a schema node inside the profile. I would not change anything. Just allow the namespace elements to be directly in the profile element.

It seems like some of the global parameters don't make sense to override (e.g., registry, oo-skeleton-path, etc.).

Agreed.

I'm assuming (for now) that profile *.df files will be found in ./data/<profile-name>/.

No, I don't think this is the case. Ovidiu already added support for specifying a custom path for the .df. This should be used unchanged. In other words, we don't need any changes to how the namespace nodes are processed in a profile.

Greg Shah wrote:

I'm assuming (for now) that profile *.df files will be found in ./data/<profile-name>/.

No, I don't think this is the case. Ovidiu already added support for specifying a custom path for the .df. This should be used unchanged. In other words, we don't need any changes to how the namespace nodes are processed in a profile.

I think that is a good idea for the default location: if importFile is not specified then the default value should be ./data/<profile-name>/<namespace>.df. For the unnamed profile the .df will be located directly into ./data/, by default.

Yes, the syntax is OK.

• there is the 'global' profile which is used implicitly
• the 'global' profile also sets defaults for any explicit profile
• each profile has the same syntax as the 'global' profile (so the namespace nodes would be added in a schema node)
• when reading a profile, first the 'global' configuration is created and after that the profile is just merged in the 'global' configuration

This way, we can work with the same Configuration instance, and other FWD conversion code will require no changes.

• 'merge' mode, where the profiles are merged and everything is converted together.
• individual mode, where FWD runs conversion for each specified profile, sequentially (same way as you would run individual ConversionDriver commands in a script, to run each profile).

When running in single profile mode, the profile will need to overwrite default parameter and file-set nodes, and merge namespace nodes to the global schema, correct?

each profile has the same syntax as the 'global' profile (so the namespace nodes would be added in a schema node)

This is a change from the current approach. I guess the benefit of consistency may outweigh the short term interest in less change. Let's go ahead.

I'm good with the other ideas. I think the multi-profile individual mode will not be used too often. I only see 2 scenarios:

• Can't run the entire project at once because of some kind of memory constraint. This would be a kind of bug that needs to be fixed anyway, so I don't see this as a long term need.
• Early runs of the conversion where we want to allow a wide range of the project to be checked without aborting on each problem. I think we are going to make conversion more tolerant of failures, but still this could have some benefit.

Is there some other use case I'm missing?

When running in single profile mode, the profile will need to overwrite default parameter and file-set nodes,

Correct.

and merge namespace nodes to the global schema, correct?

We already handle the namespace parts today for this mode. Do we need to change it?

At least the syntax in note 5 is no longer valid, right?

Right.

The profile node has only name attribute in #6256-8.

The SchemaConfig$Profile has a isDefault, which suggests that there can be a isDefault attribute at the profile node. Do we need this? I can't find any reference in redmine related to it, but there is SchemaConfig.getDefaultProfile code which uses this to determine the default profile.

The namespace should assume default="true" so that it doesn't have to be duplicated. But it should be possible to define default="false" to establish a logial database but only load it for specific files (via directory or file-level hints). We support this today. Initially, Ovidiu's changes required default="true" to be explicit but he changed that to be assumed since it is the common case.

Greg Shah wrote:

The namespace should assume default="true" so that it doesn't have to be duplicated. But it should be possible to define default="false" to establish a logial database but only load it for specific files (via directory or file-level hints). We support this today. Initially, Ovidiu's changes required default="true" to be explicit but he changed that to be assumed since it is the common case.

I'm not talking about the namespace. I'm talking about the schema/profile node. Current code assumes there can be a <profile default = "true"/> attribute.

The SchemaConfig$Profile has a isDefault, which suggests that there can be a isDefault attribute at the profile node. Do we need this? I can't find any reference in redmine related to it, but there is SchemaConfig.getDefaultProfile code which uses this to determine the default profile.

Sorry, I was thinking about the namespace. I don't recall the reason for this.

I don't think there is a hard requirement for that at the profile level.

Do you see a need to make the profile/schema node optional? In this case, it would be inherited from the global config.

PS: I'm not working on the multi-profile mode (either merge or list), this was just an idea.

Do you see a need to make the profile/schema node optional? In this case, it would be inherited from the global config.

Yes, this makes sense.

PS: I'm not working on the multi-profile mode (either merge or list), this was just an idea.

The multi-profile merge mode is needed as part of this task. The multi-profile individual mode is a useful addition, but is not required.

Greg Shah wrote:

The multi-profile merge mode is needed as part of this task.

Understood.

Greg Shah wrote:

The SchemaConfig$Profile has a isDefault, which suggests that there can be a isDefault attribute at the profile node. Do we need this? I can't find any reference in redmine related to it, but there is SchemaConfig.getDefaultProfile code which uses this to determine the default profile.

Sorry, I was thinking about the namespace. I don't recall the reason for this.

• if a profile is set as default, then automatically load the default profile (be it runtime or conversion).
• if multiple profiles are set as default, abend
• if no profile is set as default, let the global configuration be the default, unless one is specified at command line. Currently it uses the first 'schema/profile', a 'random' choice, if no default profile is set. I don't like this, it makes more sense to default to the global config. Let the user explicitly choose what needs to run.

Constantin Asofiei wrote:

• if no profile is set as default, let the global configuration be the default, unless one is specified at command line. Currently it uses the first 'schema/profile', a 'random' choice, if no default profile is set. I don't like this, it makes more sense to default to the global config. Let the user explicitly choose what needs to run.

Another addition: if a profile is set as default, command line can still override it.

It is a good plan.

Greg Shah wrote:

The multi-profile merge mode is needed as part of this task.

This will be only for conversion. Runtime part is tricky, especially the schema part.

A useful addition (medium term, not immediately) would be to be able to load multiple DF files into a single schema namespace. Rather than concatenating the DF files together manually or maintaining a script to do so, the SchemaLoader could be configured to parse multiple DF files into a single, logical schema. This could be represented as one or more import-file elements as children of the namespace element, rather than the current importFile attribute of namespace. We could leave the importFile attribute intact for backward compatibility or deprecate it, but the preferred way to define a schema namespace would be with the new import-file element. If no importFile attribute or import-file child element is defined, the namespace would default to the current approach of just assuming the name attribute of the namespace element indicates the root name of the DF file, located in the data subdirectory of the project.

For example:

<namespace name="federated">
   <import-file path="[optional/relative/location/of/]shard1.df" />
   <import-file path="[optional/different/relative/location/of/]shard2.df" />
</namespace>

Eric: I think the shard idea makes good sense. We have multiple customers using this approach in OE today. The idea should probably be in its own task.

• multiple profiles are allowed (only for conversion time), and they can be specified via -pprofile1 -pprofile2 at command line or as default=true at the p2j.cfg.xml profile
• if no command-line -p argument, then p2j.cfg.xml is checked for one or more default profiles. If none set, the global config is used.
• if one or more profiles are determined as default, start with the pristine global config, and merge all the profiles into this:
  • parameters are overwritten, in the order of how the profiles are defined in p2j.cfg.xml
  • the file-set used is the one computed from the profiles - each profile computes its files individually, and an explicit list of files is returned. The global file-set is not used!
• schema namespaces are merged from all profiles, and are used instead of the global schema. WARNING: if there is a namespace clash and the schema is not the same for all namespaces sharing this name (over all used profiles), then ... weird things will happen.
• if a profile has no schema defined, then there will be no schema used for that profile (I do not default to the global schema)

When I said the multiple profile merge was needed, I meant a different idea.

Each profile can be run standalone. But when you run multuple profiles as a batch, we need the settings to remain intact. The files specified for the profile should be parsed/converted with the same settings as if the profile was run standalone. But the overall list of files processed will be the aggregate of all the profiles selected.

parameters are overwritten, in the order of how the profiles are defined in p2j.cfg.xml

No, we don't want this. The settings for a given file will depend on the specific profile it is in + the global settings, that is all.

schema namespaces are merged from all profiles,

No, this won't work. The idea is the same as for the rest of the parameters. The files specified for that profile must get just the schemata that are specified in that profile.

Greg Shah wrote:

The files specified for the profile should be parsed/converted with the same settings as if the profile was run standalone.

What if a file is part of multiple profiles? I think we see this in a current project. Also, if there is any namespace (or schema file) conflict, it will not work.

What you describe can't be achieved easily... it means switching the conversion context for each file, to set the schema, propath settings, and more that I can't think about without digging into the code.

What if a file is part of multiple profiles?

The profiles are intended to be mutually-exclusive. We can define a rule that the first profile wins (and print a warning), if a file is specified more than once.

What you describe can't be achieved easily... it means switching the conversion context for each file, to set the schema, propath settings,

Yes, I understand. That is the intention of this task.

My idea was to add an extra layer at the ConversionDriver level, which would associate the profile with some sub-set of total file list. As each file is processed, the proper profile would be used and all schemata/configuration would come from that profile.

Greg Shah wrote:

My idea was to add an extra layer at the ConversionDriver level, which would associate the profile with some sub-set of total file list. As each file is processed, the proper profile would be used and all schemata/configuration would come from that profile.

Thanks, I think I may be able to make it work for the front phase. But the conversion part may be tricky.

We can defer any conversion issues until a second phase of work. The front end part is what is needed this week.

Greg Shah wrote:

We can defer any conversion issues until a second phase of work. The front end part is what is needed this week.

I think it works, kind of ugly IMO, but with minimal code for the front phase. I don't see a way to switch profiles without explicitly 'injecting' code like 'cfg.withFileProfile(file)', which does all the underlying work of switching the Configuration and SchemaConfig.

I'll test and commit tomorrow.

1. a merged configuration with all schemas in one database, named main_db
2. a multi configuration with all schemas spread into multiple databases, named main_db (which is a different schema than the above), multi_1, multi_2, and multi_3

And there is a "common" database between the 2 named common_db.

• data/merged which contains just the main_db
• data/multi which contains the main_db, multi_1, multi_2, and multi_3
• data/merged_multi_common which contains the common_db

What would the p2j.cfg.xml configuration look like? Right now it looks like:

   <schema>
      <profile name="merged" default="true">
         <namespace name="standard" />

         <namespace
            name="common_db" 
            default="false" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
         <namespace
            name="main_db" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
      </profile>

      <profile name="multi" default="false">
         <namespace name="standard" />

         <namespace
            name="common_db" 
            default="false" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
         <namespace
            name="main_db" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
         <namespace
            name="multi_1" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
         <namespace
            name="multi_2" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
         <namespace
            name="multi_3" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
         </namespace>
      </profile>

My build.xml goes through the effort of copying the correct set of DF files from either data/merged or data/multi to the data/ directory to be alongside the common_db.df file, which is kept in that location.

I'd like to just place the DF files as mentioned, and leave them be during the conversion process. What should my configuration look like?

As far as the dump files, there is a singular collection of all the dump files, and there are symbolic links to the same collection for dump/multi_1/, dump/multi_2/, and dump/multi_3/ so I don't believe that aspect of configuration should change.

Our initial support for multi-profile conversion needs to be enhanced. FWD supports conversion of separate profiles, but it does not allow merging all these converted profiles (with one or more of them representing an application module) and run them together, in the same FWD server.

• #6407 is part of the solution. (name_map.xml in 'extension jars')
• We need to handle the fact that per profile state like registry.xml and all the .p2o/.schema needs to be merged in some way.
• Different profiles may reference overlapping sets of OO 4GL classes. We probably need some concept of which profile is authoritative for these classes so that they can be only converted once. Doing so is a problem because the only mapping we have from 4GL OO features to Java OO features is calculated during parsing/conversion itself but the dependent projects must have access to this in order to convert. The current approach converts overlapping OO 4GL classes multiple times and has no way to merge them into a single result.

The objective here is to have two or more completely different converted applications which may or may not use the same schema/database, and we want to:

• Convert and compile each one independently, with the minimum dependencies on the others.
• Run them in the same FWD server at runtime with no duplication of classes or conversion artifacts/state.

As one customer notes: in Java this can be accomplished easily by just referencing a jar in your development environment. For that to work, we would have to package up enough of the conversion artifacts/state to allow it to be referenced in another "application". Doing this would make very large organizations able to split their development across tens or even hundreds of different development groups without each one having to convert the entire set of all dependent applications. This is a critical objective.

What would be required is that the dependencies would be converted first. Then the jar(s) (which have the converted code) would need to be available. Crucially, some amount of conversion knowledge about how the 4GL dependencies converted would be required. For example, we would need to know how each externally visible method and data member mapped to the converted version. Some amount of this (maybe all?) could be read from our class and method annotations (e.g. LegacySignature).

There is a problem when the 'exclude' filter is used to not add files to conversion. During pre-scan phase, the filter is not checked, but instead a physical file on disk (based on the configured PROPATH). And, that file is used (if found), even if is excluded via the profile 'exclude' filter.

These profiles can only be used in the default ConversionDriver mode which is ListType.CMD_LINE. For any of the other modes even ListType.FILESET, the filesets defined in the p2j.cfg.xml profiles are ignored. This is true even if the -p is specified on the command line. I think we should allow the -z (fileset mode) to honor any specified profile(s).

Created task branch 6256a from latest trunk/15941.

Constantin Asofiei wrote:

There is a problem when the 'exclude' filter is used to not add files to conversion. During pre-scan phase, the filter is not checked, but instead a physical file on disk (based on the configured PROPATH). And, that file is used (if found), even if is excluded via the profile 'exclude' filter.

I did a test where I created a start1.p file and excluded it in zfile_set.txt. It got taken out from conversion so this one worked out. From what I understand, is the issue that we have a propath (e.g. folder1) and this case:

1. X ./abl/start1.p # zfile_set.txt
2. conversion finds ./abl/folder1/start1.p but it is not excluded
3. The exclude filter should take the propath into account when excluding files.

For #6256-53, I think the fix needs to involve the p2j.cfg.xml (the propath and basepath parameters). In FileListFactory.processFileSet(), it is required to add the basepath and propath to the file one by one and check if the file exists. The problem comes with propath like ${P2J_HOME}/abl/common/ which define the absolute path instead of the relative path from the ExplicitFileList of the existent references.

As I see it, the zfile_set.txt should look like this:

D ./abl/ *.[pPwW]

X test1.p

No, X directive works only with ./abl/ paths - there can't be relative paths.

During pre-scan phase, the filter is not checked, but instead a physical file on disk (based on the configured PROPATH). And, that file is used (if found), even if is excluded via the profile 'exclude' filter.

Wasn't this solved in #7255?

• -p is an option that can be used to specify a profile, is there any documentation on how to create such profile and what it should contain?
• how should I define additional filesets in p2j.cfg.xml?

Dănuț Filimon wrote:

I am trying to understand how p2j.cfg.xml is used in this context, I've found the following documentation https://proj.goldencode.com/projects/p2j/wiki/Chapter_6_Project_Setup#Configuration-Reference and my questions are:

• -p is an option that can be used to specify a profile, is there any documentation on how to create such profile and what it should contain?
• how should I define additional filesets in p2j.cfg.xml?

I no longer need answers to this, there are examples in #6256-4 and #6256-8. I will continue investigating.

Dănuț Filimon wrote:

Dănuț Filimon wrote:

I am trying to understand how p2j.cfg.xml is used in this context, I've found the following documentation https://proj.goldencode.com/projects/p2j/wiki/Chapter_6_Project_Setup#Configuration-Reference and my questions are:

• -p is an option that can be used to specify a profile, is there any documentation on how to create such profile and what it should contain?
• how should I define additional filesets in p2j.cfg.xml?

I no longer need answers to this, there are examples in #6256-4 and #6256-8. I will continue investigating.

Those examples are not actually the right ones. I've checked the profiles defined for a customer application and those match the ones mentioned in #6256-41. Right now I am not sure of defining filesets in a profile and reading them and how should a profile be specified when using -p option.

I've experimented and the following worked:

   <profile name="test" default="true">
     <schema>
      <namespace
         name="hotel" 
         importFile="data/hotel.df" 
         default="true" >
         <parameter name="ddl-dialects" value="h2,postgresql,mariadb" />
         <dialect-specific name="h2">
            <parameter name="collation" value="en_US@iso88591_fwd_basic" />
         </dialect-specific>
         <dialect-specific name="postgresql">
            <parameter name="udf" value="sql" />
         </dialect-specific>
         <dialect-specific name="mariadb">
            <parameter name="udf" value="sql" />
         </dialect-specific>
      </namespace>
     </schema>
      <file-set>
         <directory recursive="true" path="./abl/" spec="(*.[pPwWtT]|*.cls)" />
         <exclude filter="./abl/possenet/*" />
      </file-set>
   </profile>

I managed to combine the previous notes and an example configuration from a customer and create a profile for hotel. I'll be experimenting more, the -ptest option is how I should be able to use the profile.

Please plan to make additions to our documentation to explain this syntax.

Greg Shah wrote:

Please plan to make additions to our documentation to explain this syntax.

I will do so in this wiki https://proj.goldencode.com/projects/p2j/wiki/Chapter_6_Project_Setup#Profiles.

• parameters
• fileSet
• schema
• includeRemap
• name
• isDefault
• savedStates
• paths
• fileList

When a profile is used using the -p option and we also use ListType.FILESET (-z), it should behave like -z was not specified (just like ListType.CMD_LINE) or should both the -z and -p be honored (processing the -z fileset and also the fileset of the profile)? Some file may be part of both filesets, but I think the profiles should be used as a way to exclude/add more files from the existent -z fileset,.

When a profile is used using the -p option and we also use ListType.FILESET (-z), it should behave like -z was not specified (just like ListType.CMD_LINE) or should both the -z and -p be honored (processing the -z fileset and also the fileset of the profile)? Some file may be part of both filesets, but I think the profiles should be used as a way to exclude/add more files from the existent -z fileset,.

Good question.

In our near future, we will implement the concept that a profile is equivalent to an application (or a module). The intention is that a profile is standalone AND when we add #7169, we will provide encoding of dependencies between profiles (e.g. profile A depends on profile B which depends on profiles C and D). This will drive "app by app" conversion on a managed basis which conversion of one application/module (profile) completing before profiles that are dependent upon it.

So we intend for profiles to be self-contained definitions of applications/modules. I prefer not to mix random -z filesets into it. We should ignore a -z when one or more profiles are specified.

Constantin: Thoughts?

So we intend for profiles to be self-contained definitions of applications/modules. I prefer not to mix random -z filesets into it. We should ignore a -z when one or more profiles are specified.

+ add a log/message that this will happen. This will avoid confusion on "why my -z doesn't work"?

Alexandru Lungu wrote:

So we intend for profiles to be self-contained definitions of applications/modules. I prefer not to mix random -z filesets into it. We should ignore a -z when one or more profiles are specified.

+ add a log/message that this will happen. This will avoid confusion on "why my -z doesn't work"?

Yes. Good point.

Greg Shah wrote:

When a profile is used using the -p option and we also use ListType.FILESET (-z), it should behave like -z was not specified (just like ListType.CMD_LINE) or should both the -z and -p be honored (processing the -z fileset and also the fileset of the profile)? Some file may be part of both filesets, but I think the profiles should be used as a way to exclude/add more files from the existent -z fileset,.

Good question.

In our near future, we will implement the concept that a profile is equivalent to an application (or a module). The intention is that a profile is standalone AND when we add #7169, we will provide encoding of dependencies between profiles (e.g. profile A depends on profile B which depends on profiles C and D). This will drive "app by app" conversion on a managed basis which conversion of one application/module (profile) completing before profiles that are dependent upon it.

So we intend for profiles to be self-contained definitions of applications/modules. I prefer not to mix random -z filesets into it. We should ignore a -z when one or more profiles are specified.

Constantin: Thoughts?

• if the profile has a fileset, then that is used
• if the profile has no fileset, then whatever is at the command line is used

So yes, we will not mix any kind of filesets, from command line or between profiles.

I committed 6256a/15492. Ignore the -z fileset, but only when there's a profile fileset specified.

Constantin Asofiei wrote:

If profiles are used, then:

• if the profile has a fileset, then that is used
• if the profile has no fileset, then whatever is at the command line is used

• When there is a default profile defined, but the -p option is not used to specify the profile.

• includeRemap: Should be similar to parameter: <includeRemap name="?" value="?" />, but what is it used for? Can I have an example?
• paths: ?
• savedStates: <state />. How should this be defined and work?
• fileList: this is stored as a Set in ProfileConfig, so I am also not sure how it should be set here.

Dănuț Filimon wrote:

I also want to add the following scenario:

• When there is a default profile defined, but the -p option is not used to specify the profile.

This works with the latest 6256a, but I need to clarify if the default profile is used even when -p is not specified.

Dănuț Filimon wrote:

Dănuț Filimon wrote:

I also want to add the following scenario:

• When there is a default profile defined, but the -p option is not used to specify the profile.

This works with the latest 6256a, but I need to clarify if the default profile is used even when -p is not specified.

Yes, IIRC the global/default profile is inherited by the individual profiles - and these override any config set there.

Dănuț Filimon wrote:

I updated https://proj.goldencode.com/projects/p2j/wiki/Chapter_6_Project_Setup#Profiles, but I am not sure how to define:

• includeRemap: Should be similar to parameter: <includeRemap name="?" value="?" />, but what is it used for? Can I have an example?

I don't have any examples for this and don't recall it... Greg?

• paths: ?
• savedStates: <state />. How should this be defined and work?

This is not used in p2j.cfg.xml - this is used 'by runtime', to save state at each profile.

• fileList: this is stored as a Set in ProfileConfig, so I am also not sure how it should be set here.

These are the 'resolved files' from the fileSet. Used by runtime, no p2j.cfg.xml configuration. The point is, as with current trunk: you can run conversion 'for all profiles', and each file finds the profile where it belongs, and activates the profile's configuration. So if you run with 'default profile', then each file with gets its configuration from the associated profile.

Constantin Asofiei wrote:

• paths: ?

This is the PROPATH for the profile.

I updated https://proj.goldencode.com/projects/p2j/wiki/Chapter_6_Project_Setup#Profiles, but I am not sure how to define:

• includeRemap: Should be similar to parameter: <includeRemap name="?" value="?" />, but what is it used for? Can I have an example?

I don't have any examples for this and don't recall it... Greg?

I don't know what that is either. Where is it used in the FWD code?

includeRemap is from:

revno: 11338.1.8
author: Eric Faulhaber <ecf@goldencode.com>, Greg Shah <ges@goldencode.com>
committer: Igor Skornyakov <ias@goldencode.com>
branch nick: 4069a
timestamp: Wed 2019-06-19 20:54:49 -0400
message:
  Added preprocessor support for remapping/removing the prefix of an include file reference

Is used in p2j.preproc.FileScope.resolveFileName

I don't remember that code. It isn't currently used in the project associated with #4069.

Still, if it was needed once for preprocessing, then it will probably be needed again.

This is a decent explanation:

   /**
    * Mapping of include file path prefixes as found in source file include references, to new
    * prefixes, which will allow include files to be found in the conversion project directory
    * hierarchy. The replacement is commonly empty string, to make an absolute file reference
    * in the original code into a relative file reference. However, it could be something else.
    * This facility is not meant to replace the PROPATH, but rather to adjust for cases (e.g.,
    * absolute include file references) where the source code might use include file references
    * which are specific to a particular deployment which is not general purpose, and the PROPATH
    * alone is not enough.
    */
   public static class IncludeRemap

I only see it used in one of the FileScope constructors (which calls options.getIncludeFilePrefixMap()). The format is name=oldpath and value=newpath.

• includeRemap: #6256-76 is really helpful, I also found that the tag is not <includeRemap name="?" value="?" />, but <include-mappings name=oldpath value=newpath /> and is defined in Configuration.TAG_INCLUDE_MAPPINGS. When I tried it, it failed with the error

       [java] Caused by: java.lang.IllegalStateException: Could not find default setter 'setName()' for property 'name' in class com.goldencode.p2j.cfg.Configuration$IncludeRemap (setProperty() may need to be overriden by subclass in order to handle particular cases)
  

  However, I did find #4081-14 after searching for the include-mappings and it worked.
• paths is setup at runtime if there is a propath parameter defined.
• savedStates is set at runtime to save the state at each profile.
• fileList is used at runtime and contains the resolved files from the fileSet

I have enough information to complete the ProfileConfig wiki.

Constantin Asofiei wrote:

No, X directive works only with ./abl/ paths - there can't be relative paths.

During pre-scan phase, the filter is not checked, but instead a physical file on disk (based on the configured PROPATH). And, that file is used (if found), even if is excluded via the profile 'exclude' filter.

Wasn't this solved in #7255?

The goal of #7255 was to automatically include in conversion those .cls files which does not appear in the conversion list, but there are classes specifically included in the conversion list that reference these culprit classes.

Example:

class oo.Foo inherits oo.Bar:

In the conversion list abl/oo/Foo.cls is included but not abl/oo/Bar.cls.

With #7255 at parsing phase abl/oo/Bar.cls will be automatically included in conversion.

Talking about the case when a class is specifically excluded list, but still there is a reference in on of the converted files. I suppose it should be treated as if it wasn't present in the conversion list at all, at add it in parsing phase. Otherwise we will get an error in a later phase.

Code Review Task Branch 6256a Revisions 15942 and 15943

The change is good.

Are you sure that the requirements of #6256-47 and #6256-50 are already handled? I didn't think that #7255 resolved #6256-47. I didn't think we could use the -p to run full conversion for an arbitrary set of profiles (#6256-50).

Greg Shah wrote:

Are you sure that the requirements of #6256-47 and #6256-50 are already handled? I didn't think that #7255 resolved #6256-47. I didn't think we could use the -p to run full conversion for an arbitrary set of profiles (#6256-50).

I could not find an issue when attempting to reproduce #6256-47, so I assume this was fixed. As for #6256-50, this only handles the ListType.FILESET scenario and not the other modes. As long as the profile fileset is defined properly, -z is redundant in this case.

There is a problem when the 'exclude' filter is used to not add files to conversion. During pre-scan phase, the filter is not checked, but instead a physical file on disk (based on the configured PROPATH). And, that file is used (if found), even if is excluded via the profile 'exclude' filter.

Constantin: Do you have a specific recreate Danut can test?

These profiles can only be used in the default ConversionDriver mode which is ListType.CMD_LINE. For any of the other modes even ListType.FILESET, the filesets defined in the p2j.cfg.xml profiles are ignored. This is true even if the -p is specified on the command line. I think we should allow the -z (fileset mode) to honor any specified profile(s).

I agree that as discussed in #6256-63 and #6256-66 we don't need to support profiles and an explicit -z fileset.

Looking at the code, I think it is pretty confusing currently. If one specifies -p, it is ignored in BLACKLIST, WHITELIST and FILESPEC modes.

Perhaps we need to think about this differently. When -p is specified, we should consider this a new PROFILE mode that is mutually exclusive with -x, -f, -s or -z modes as well as the default CMDLINE mode. Please make changes to implement that. If -p is specified, it is honored and any other mode is ignored (with a warning message). Document it as a new PROFILE mode.

Also: What is the status of specifying more than one profile name? Is that fully working?

Greg Shah wrote:

There is a problem when the 'exclude' filter is used to not add files to conversion. During pre-scan phase, the filter is not checked, but instead a physical file on disk (based on the configured PROPATH). And, that file is used (if found), even if is excluded via the profile 'exclude' filter.

Constantin: Do you have a specific recreate Danut can test?

This is the #7255 problem which was fixed.

Committed 6256a/15944. Added PROFILE mode.

Currently looking into the error from #6256-84.

Greg, the issue with #6256-84 is that I've defined the following namespace for both profiles:

      <namespace
         name="hotel" 
         importFile="data/hotel.df" 
         default="true" >
         <parameter name="ddl-dialects" value="h2,postgresql,mariadb" />
         <dialect-specific name="h2">
            <parameter name="collation" value="en_US@iso88591_fwd_basic" />
         </dialect-specific>
         <dialect-specific name="postgresql">
            <parameter name="udf" value="sql" />
         </dialect-specific>
         <dialect-specific name="mariadb">
            <parameter name="udf" value="sql" />
         </dialect-specific>
      </namespace>

I don't think two profiles should have the same namespace defined (with the same name).

I don't think two profiles should have the same namespace defined (with the same name).

Correct. Such a thing isn't needed because any common namespace can (and should) be defined at the top level, instead of inside a profile.

Greg Shah wrote:

I don't think two profiles should have the same namespace defined (with the same name).

Correct. Such a thing isn't needed because any common namespace can (and should) be defined at the top level, instead of inside a profile.

The top level schema is not picked up when using multiple profiles, I will be investigating this issue next.

The issue is that there are three profiles and the two profiles that do not define a namespace for the hotel, do not pick it up even if it is defined in the top level schema. In the end, the ddls do not generate for the hotel database. The fix should include checking the profile stored in SchemaConfig.profiles collection with the key null and using the NsConfig or simply duplicating the NsConfig for each profile when it is created.

The issue is related to TransformDriver.createDatabaseList() where the "standard" namespace is not taken into consideration because it uses Configuration.dbProfile.

I fixed the problem by removing the bit of code that checks for isMultiProfile(), this was added in:

revno: 14484 [merge]
committer: Constantin Asofiei <ca@goldencode.com>
branch nick: trunk
timestamp: Wed 2023-02-15 07:50:28 -0500
message:
  Merge of branch 6129c - this contains more than one year work to convert, fix runtime issues, implement MariaDB support and FWD performance fixes.  Some important changes are:
  - major overhaul of the OO conversion, plus OO runtime fixes
  - fixes for the FWD JavaOpenClient support
  - lots of memory leak and performance fixes
  - improved profile support for conversion
  - persistence-related fixes
  Refs #6129 and others.

In src/com/goldencode/p2j/convert/TransformDriver.java, I removed this code:

+         if (cfg.isMultiProfile())
+         {
+            cfg.withDbProfile(schema, (profile) ->
+            {
+               String profileMetaName = config.getMetadata().getName();
+
+               // omit meta if so required 
+               if (omitMeta && schema.equals(profileMetaName))
+               {
+                  return;
+               }
+               
+               String fileName = config.getSchemaFileName(schema, extension);
+               if (new File(fileName).exists() && !toReturn.contains(fileName))
+               {
+                  // collect ONLY existing files
+                  toReturn.add(fileName);
+               }
+            });
+            continue;
+         }

Constantin, the standard namespace is ignored because it looks in Configuration.dbProfile where it only finds hotel. Was ignoring the standard namespace intentional when using multiple profiles?

Committed 6256a/15945. Fixed the two issues I found when converting hotel using multiple profiles.

Code Review Task Branch 6256a Revisions 15944 and 15945

If a profile contains any schema at all, then the top-level namespaces are ignored. I think that is a problem because customers will have to duplicate definitions in any profile that just wants to add something.

Otherwise the changes are good.

Greg Shah wrote:

If a profile contains any schema at all, then the top-level namespaces are ignored. I think that is a problem because customers will have to duplicate definitions in any profile that just wants to add something.

I'll test this scenario and update the task with the results. If everything works as expected, I'll put it in Internal Test.

The global schema should be used by a profile only when none of the active profiles have a schema defined, this is the issue with the SchemaConfig.

• When using a profile without a schema, the global schema should be copied to that profile.
• When using multiple profiles and none use a schema, the global schema should be copied to all profiles.
• When using multiple profiles and only one has a schema, the schema should be copied to the other profiles.
• When using multiple profiles with different schemas, the profiles should remain the same.
• When using multiple profiles with different schemas, but one is not configured - ?

If profiles are created and can be used without a schema, the last scenario will become confusing when we want to introduce one internally.

A problem right now is that when a profile defines a schema, the SchemaLoader.loadDefaults() that is called when creating a SchemaDictionary in the Schema Annotations phase is using the global schema. We should not convert a schema that is not used and this is what I am trying to avoid right now.

The issue with the database is actually related to how the default profile in SchemaCOnfig is set, the root cause is Configuration.withFileProfile() which changes the default profile to null even when the searched file is not found with the global propath (it happens when it searches for files that are not part of the abl folder - cvt/data/standard, dmo files - Dmo.java.jast).

I am working on a fix.

Anything at the global level should be assumed to be present for all profiles. Anything in a profile should be added to the global schemata when that profile is processing, unless the database name is the same as one in the global schemata, in which case it should override.

Anything defined in one profile should never have an affect on any other profile.

I think your description above seems to break these concepts.

Code Review Task Branch 6256a Revision 16022

My only question is why would the metadata schema be bypassed in profile mode. I understand it would be expected to be defined in the global namespace, but importantly it must also be processed for all schemata whether they are defined globally or in a profile, right?

Greg Shah wrote:

My only question is why would the metadata schema be bypassed in profile mode. I understand it would be expected to be defined in the global namespace, but importantly it must also be processed for all schemata whether they are defined globally or in a profile, right?

The "standard" namespace (default) has a schema file, but no database profile for it. When using 2 or more profiles, we are facing the issue with the missing database profile (dbProfile is checked for the "standard" database which is not defined), but when using 1 profile or none we are not checking for the database, just for the actual schema file (dict, schema, p2o). The changes ensure that we fallback to the normal check when we fail to find a matching dbprofile.

I got a message from Razvan that he converted the ChUI project, but the "convert" command was skipped. This is because there is no fileset defined in the profile.

Greg, similar to the schema, I should default to the global configuration in such cases and use the command line arguments (app.4gl.src and app.4gl.pattern). But what should I do when a profile with no fileset is in use and there are different ways to get the file-set?

and there are different ways to get the file-set?

What do you mean?

Greg Shah wrote:

and there are different ways to get the file-set?

What do you mean?

ChUI project uses the path and spec in the command line, while other applications use -z.

I got a message from Razvan that he converted the ChUI project, but the "convert" command was skipped. This is because there is no fileset defined in the profile.

What profile? The ChUI application doesn't use profiles.

ChUI project uses the path and spec in the command line,

Is there a reason this should stop working?

while other applications use -z.

I expect -z to work, unless -p is used.

Greg Shah wrote:

I got a message from Razvan that he converted the ChUI project, but the "convert" command was skipped. This is because there is no fileset defined in the profile.

What profile? The ChUI application doesn't use profiles.

This is the command:

   <!-- convert 4GL code from ${app.4gl.src} into ${src}, using ${app.4gl.pattern} pattern -->
   <target name="convert" 
           depends="init, srcnew.precompile, check-file-cvt-list, check-file-ignore-list" 
           description="Convert the 4GL source code to Java source code, using a pattern as input." 
           unless="no-list-mode">
      <record name="cvt_${LOG_STAMP}.log" action="start"/>
      <java classname="com.goldencode.p2j.convert.ConversionDriver" 
            fork="true" 
            failonerror="true" 
            dir="${basedir}">
         <jvmarg value="-server"/>
         <jvmarg value="-Xmx${conversionHeap}"/>
         <jvmarg value="-XX:-OmitStackTraceInFastThrow"/>
         <jvmarg value="-DP2J_HOME=${p2j.home}"/>
         <arg value="-I"/>
         <arg value="-S"/>
         <arg value="-d2"/>
         <arg value="-P${app.db.profile}" />
         <arg value="f2+m0+cb"/>
         <arg value="${app.4gl.src}"/>
         <arg value="${app.4gl.pattern}"/>
         <classpath refid="convert.classpath"/>
      </java>
      <record name="cvt_${LOG_STAMP}.log" action="stop"/>
   </target>

   <profile name="default" default="true">
      <schema>
         <namespace name="standard" importFile="data/standard.df" default="false" />

         <namespace name="<removed>" importFile="data/<removed>.df">
            <parameter name="ddl-dialects" value="h2,postgresql" />
            <parameter name="generate-pojos" value="true" />
            <dialect-specific name="h2">
               <parameter name="collation" value="en_US@iso88591_fwd_basic" />
            </dialect-specific>
         </namespace>
      </schema>
   </profile>

   <!-- Application specific DB handling properties -->
   <property name="app.db.profile"       value="default" />

ChUI project uses the path and spec in the command line,

Is there a reason this should stop working?

while other applications use -z.

I expect -z to work, unless -p is used.

1. Make the namespaces global and remove the profile from the command.
2. Include the file-set in the profile and remove the unnecessary parameters (it will need more configuration changes).

I think it's better to create a testing for 6256a issue to have a look at all projects.

EDIT: Removed project mentions

I think we should modify the profile to do one of the following:

1. Make the namespaces global and remove the profile from the command.
2. Include the file-set in the profile and remove the unnecessary parameters (it will need more configuration changes).

Yes, either of these is a correct approach.

I think it's better to create a testing for 6256a issue to have a look at all projects.

Yes

Constantin, this is related to 5586a changes. Can you create a separate task?

I made updates to an application to position the .df files into the profile directory so that the p2j.cfg.xml would allow conversion to find them. I did this so as to not force the build.xml to move things around based upon which profile was being built. Much cleaner, much easier.

1. I could not import without them in place so I'm sure they are required for runtime (at least for import). So I need to move their archive from incremental to deploy. Not too challenging, except that their location is cvt/data/<profile>/ and we need them to be in data/ for runtime usage. This leads me to the next issue...
2. 2 I could not perform an import with the .dict and .p2o files in the profile directory next to the .df file. I had to move them into the data/ directory. Is that how it should work?

Also, the ScriptRunner wants to look for them in the application jar file, but if it doesn't falls back to looking in the actual filesystem. So while I was able to hand-crank my test with the filesystem changes above, I'm not sure where these .p2o and .dict files should be in the jar.

Roger Borrello wrote:

I made updates to an application to position the .df files into the profile directory so that the p2j.cfg.xml would allow conversion to find them. I did this so as to not force the build.xml to move things around based upon which profile was being built. Much cleaner, much easier.

However, 2 issues arose with respect to .p2o and .dict files that I found when I tried to import the database.

1. I could not import without them in place so I'm sure they are required for runtime (at least for import). So I need to move their archive from incremental to deploy. Not too challenging, except that their location is cvt/data/<profile>/ and we need them to be in data/ for runtime usage. This leads me to the next issue...
2. 2 I could not perform an import with the .dict and .p2o files in the profile directory next to the .df file. I had to move them into the data/ directory. Is that how it should work?

Also, the ScriptRunner wants to look for them in the application jar file, but if it doesn't falls back to looking in the actual filesystem. So while I was able to hand-crank my test with the filesystem changes above, I'm not sure where these .p2o and .dict files should be in the jar.

Artifacts were moved to the cvt folder in trunk/16089 (#5586), please post your errors in #10189 and I will take a look. Even better if you can provide a scenario for me to test.

Not sure about the .dict but we currently have a dependency on the .p2o for import and it certainly should have already been part of the jar.

Putting these files in the data/ directory is something that is very broken and makes no sense. We need to create a task to eliminate any dependencies on a hard coded data/ directory and instead allow the path to the dump files to be parameterized.

Greg Shah wrote:

Not sure about the .dict but we currently have a dependency on the .p2o for import and it certainly should have already been part of the jar.

Putting these files in the data/ directory is something that is very broken and makes no sense. We need to create a task to eliminate any dependencies on a hard coded data/ directory and instead allow the path to the dump files to be parameterized.

Let me get back on this, because it was taking a while to re-jar and I want to test with a new jar after I made an update to the build.xml:

   <target name="prepare.compile" 
           depends="prepare.embedded, prepare.srcnew, jasper.compile, admin.compile" 
           description="Sets up the compilation environment (build directories, distribution directories, position files)." >
...
      <copy todir="${build.home}/classes/${data.rel}/${app.db.profile}">
         <fileset dir="${cvt.home}/${data.rel}/${app.db.profile}" includes="*.dict" />
         <fileset dir="${cvt.home}/${data.rel}/${app.db.profile}" includes="*.p2o" />
      </copy>
   </target>
...
      <zip destfile="${archive.dist_destfile}" >
         <zipfileset dir="${basedir}/${data.rel}/${app.db.profile}/" 
                     includes="*.df" 
                     prefix="${data.rel}" />
         <zipfileset dir="${basedir}/${cvt.rel}" 
                     includes="${data.rel}/*.dict,
                               ${data.rel}/*.p2o" />
         <zipfileset dir="${basedir}/${cvt.rel}/${data.rel}/${app.db.profile}" 
                     includes="*.dict,
                               *.p2o" 
                     prefix="${data.rel}" />
...

Correct me if I am wrong, but there are 2 solutions... 1. put (.dict/.p2o) it in the correct place in the jar, and 2. put them in the correct place on the file system, and the correct place should be in the profile directory.

Yes, the filesystem approach is for dev environments and the jar would be used in prod.

Greg Shah wrote:

Yes, the filesystem approach is for dev environments and the jar would be used in prod.

Right now I'm zipping the files into the deploy archive. Would the debug archive be more appropriate, since I fixed the jar via the prepare.compile task?

BTW, even though these changes are not necessary to the Hotel samples (since they don't use an actual profile directory), I would suggest we update them in terms of being samples.

If they are in the jar, we definitely DO NOT want them deployed in the filesystem.

BTW, even though these changes are not necessary to the Hotel samples (since they don't use an actual profile directory), I would suggest we update them in terms of being samples.

Of course. I always assume you are doing this.

Greg Shah wrote:

If they are in the jar, we definitely DO NOT want them deployed in the filesystem.

What about the .df file? Is that not to be deployed in the filesystem?

BTW, even though these changes are not necessary to the Hotel samples (since they don't use an actual profile directory), I would suggest we update them in terms of being samples.

Of course. I always assume you are doing this.

I was, but just wanted to socialize. I had worded it "I will" and changed it to "I would suggest" because I wanted to give you a chance to think over actually creating a profile as an example. It would be a good sample, but might lead customers to believe they always need to have a profile.

Roger Borrello wrote:

Greg Shah wrote:

If they are in the jar, we definitely DO NOT want them deployed in the filesystem.

What about the .df file? Is that not to be deployed in the filesystem?

Never. It is like source code. There is no reason to ever have it in the production environment.

BTW, even though these changes are not necessary to the Hotel samples (since they don't use an actual profile directory), I would suggest we update them in terms of being samples.

Of course. I always assume you are doing this.

I was, but just wanted to socialize. I had worded it "I will" and changed it to "I would suggest" because I wanted to give you a chance to think over actually creating a profile as an example. It would be a good sample, but might lead customers to believe they always need to have a profile.

What do you mean by a "profile directory"? Are you talking about the cvt/ directory? This absolutely should be implemented. It is a best practice and our standard build scripts should work with that approach.

If you are talking about creating a profile in p2j.cfg.xml, this is less clear and we could discuss the approach. A key point is that it is a best practice to put as much of the configuration (like file sets) into the p2j.cfg.xml even for Hotel. If the file-sets are large, then we can externalize them to files and refer to those files from a p2j.cfg.xml file set. But that is a last resort. Optimally, we want 100% of our cfg in that one cfg file (which will eventually be JSON).

Greg Shah wrote:

What do you mean by a "profile directory"? Are you talking about the cvt/ directory? This absolutely should be implemented. It is a best practice and our standard build scripts should work with that approach.

If you are talking about creating a profile in p2j.cfg.xml, this is less clear and we could discuss the approach. A key point is that it is a best practice to put as much of the configuration (like file sets) into the p2j.cfg.xml even for Hotel. If the file-sets are large, then we can externalize them to files and refer to those files from a p2j.cfg.xml file set. But that is a last resort. Optimally, we want 100% of our cfg in that one cfg file (which will eventually be JSON).

I mean setting up the application so that the ./cfg/p2j.cfg.xml includes a profile: Show exampleShow example

   <profile name="main_profile" default="true">
      <file-set filename="zfile_set.txt" />
      <schema>
         <namespace
            name="standard" 
            importFile="data/standard.df" />

         <namespace
            name="hotel" 
            importFile="data/hotel.df" 
            default="true" >
            <parameter name="ddl-dialects" value="h2,postgresql,sqlserver2012,mariadb,mariadblenient" />
            <dialect-specific name="h2">
               <parameter name="collation" value="en_US@iso88591_fwd_basic" />
            </dialect-specific>
            <dialect-specific name="postgresql">
               <parameter name="udf" value="java,sql" />
            </dialect-specific>
         </namespace>
      </schema>
   </profile>

It's a little stilted, but with it I'm able to remove all the extra convert.zset BS. I wouldn't move the hotel.df file anywhere. Just include a profile.

Dănuț Filimon wrote:

Artifacts were moved to the cvt folder in trunk/16089 (#5586), please post your errors in #10189 and I will take a look. Even better if you can provide a scenario for me to test.

The error is setup by moving the data/hotel.df file to a location like data/x/hotel.df and make the corresponding update in p2j.cfg.xml:

...
      <namespace
         name="hotel" 
         importFile="data/x/hotel.df" 
...

The issue is really with the last parameter to

Import dump from /home/rfb/projects/hotel_dev2/data/dump/hotel into hotel using ID fwd_user ...
Performing cmd=java -Xmx4g -Djava.system.class.loader=com.goldencode.asm.AsmClassLoader -Dfile.encoding=UTF-8 -Djava.locale.providers=SPI,CLDR,COMPAT -cp /home/rfb/projects/fwd/trunk_dev/build/lib/p2j.jar:/home/rfb/projects/fwd/trunk_dev/build/lib/fwdspi.jar:/home/rfb/projects/hotel_dev2/deploy/lib/hotel.jar com.goldencode.p2j.pattern.PatternEngine -d 2 dbName=\"hotel\" targetDb=\"postgresql\" url=\"'jdbc:postgresql://localhost:5432/hotel'\" uid=\"fwd_user\" pw=\"user\" dataPath=\"/home/rfb/projects/hotel_dev2/data/dump/hotel/\" emptyImp=false maxThreads=4 jdbcBatchSize=200 schema/import data/ hotel.p2o
...
ERROR:
java.lang.RuntimeException: ERROR!  Active Rule:
-----------------------
      RULE REPORT      
-----------------------
Rule Type :   INIT
Source AST:  null
Copy AST  :  null
Condition :  tenantMode.equalsIgnoreCase("tenant")
Loop      :  false
--- END RULE REPORT ---

        at com.goldencode.p2j.pattern.PatternEngine.run(PatternEngine.java:1111)
        at com.goldencode.p2j.pattern.PatternEngine.main(PatternEngine.java:2224)
Caused by: com.goldencode.ast.AstException: Error loading AST XML file/resource: data/hotel.p2o
        at com.goldencode.ast.XmlFilePlugin.loadTree(XmlFilePlugin.java:470)
        at com.goldencode.ast.AstManager.loadTree(AstManager.java:325)
        at com.goldencode.p2j.pattern.PatternEngine.processAst(PatternEngine.java:1514)
        at com.goldencode.p2j.pattern.PatternEngine.run(PatternEngine.java:1065)
        ... 1 more
Caused by: java.lang.IllegalArgumentException: Missing/bogus input file/resource data/hotel.p2o
        at com.goldencode.ast.XmlFilePlugin.loadTree(XmlFilePlugin.java:463)
        ... 4 more

This might be a self-inflicted wound, but I will say that import via the build_db.xml has the same issue. Neither is written to handle the .p2o file being in a location other than the current direction.

So the question is, if the .df file is moved, should the .p2o file be located next to it? And if so, both build_db.xml and import.sh need to be able to either accept a parameter, or PatternEngine needs to be able to find it.

The .df should stay in data/. It is only the intermediate and output artifacts that should be in cvt/data/.

I mean setting up the application so that the ./cfg/p2j.cfg.xml includes a profile

Please show the example where there is no profile and everything is at the top level to keep it simple.

I'm able to remove all the extra convert.zset BS.

This is good. Please do this without the profile (use the default file-set etc...).

Greg Shah wrote:

I'm able to remove all the extra convert.zset BS.

This is good. Please do this without the profile (use the default file-set etc...).

The current build.xml has a convert, convert.front, and convert.middle tasks, all of which specify <arg value="-P${app.db.profile}" />. I could remove that option, or move those tasks to convert.profile, convert.front.profile, and convert.middle.profile, but we'd be kind of in the same boat as before. I've been doing this on Hotel ChUI, but as soon as I move to Hotel GUI, we'll need to include a fileset that ignores the possenet. IMHO, we'd rarely be using the default file-set, where all the files in ./abl would be candidates for conversion.

An alternative I propose is to leave all the convert tasks using the <arg value="-P${app.profile}" /> (removed the .db, since that's a misnomer), and include this in the build.xml:

   <!-- Override for a differnent application profile -->
   <property name="app.profile"          value="default" />

   <profile name="default" default="true">
      <file-set filename="zfile_set.txt" />
      <schema>
         <namespace
            name="standard" 
            importFile="data/standard.df" />

         <namespace
            name="hotel" 
            importFile="data/hotel.df" 
            default="true" >
            <parameter name="ddl-dialects" value="h2,postgresql,sqlserver2012,mariadb,mariadblenient" />
            <dialect-specific name="h2">
               <parameter name="collation" value="en_US@iso88591_fwd_basic" />
            </dialect-specific>
            <dialect-specific name="postgresql">
               <parameter name="udf" value="java,sql" />
            </dialect-specific>
         </namespace>
      </schema>
   </profile>

Let me know your thoughts.

Greg Shah wrote:

The .df should stay in data/. It is only the intermediate and output artifacts that should be in cvt/data/.

The problem is when we need multiple profiles, like for the multi-db setup. If the filesystem is setup for conversion with:

./data/
./data/merged_profile/
./data/merged_profile/main.df
./data/multi_profile/
./data/multi_profile/main.df
./data/multi_profile/prices.df
./data/multi_profile/items.df

Let me know your thoughts.

I'm OK with the approach, especially if the build.xml can be common to most applications. That is an objective.

Roger Borrello wrote:

Greg Shah wrote:

The .df should stay in data/. It is only the intermediate and output artifacts that should be in cvt/data/.

The problem is when we need multiple profiles, like for the multi-db setup. If the filesystem is setup for conversion with:
[...]
The conversion should place the cvt artifacts in cvt/data/<the profile used>/main.p2o and cvt/data/<the profile used>/main.dict (since we can only build under a given profile)

Agreed. If this doesn't work, it is a bug to be fixed in FWD.

so the resultant jar needs to be setup for success so the .p2o files can be found upon import by the PatternEngine. That means the artifacts should be positioned by the prepare.compile step in the ${build.home}/classes/${data.rel} directory.

I don't like this much but I guess it makes sense. Since the application must be converted with only one of the profiles, it will work. The problem is more about when multiple profiles are active in the same application at runtime. Then it is important not to have conflicts between the profiles. You are using profiles for a mutually exclusive case here, so the conflicts are OK.

The only thing I can thing of that would be a problem is if there is a collision in .df filenames from the ./data/ and ./data/<the profile used> locations. Is it possible to have ./data/main.df and ./data/<the profile used>/main.df? Or is that counter to the intention of using profiles in this manner?

That is against the idea. If you are going to be using profiles, then they should not conflict with anything "global".

What I gather from your comments is that the multi-profile configuration of the application I'm talking about is OK to utilize my scheme of positioning the .p2o files for the jar in the correct place, but only because of the nature of the application... that the profiles are mutually exclusive.

Agreed. If this doesn't work, it is a bug to be fixed in FWD.

The data artifacts are placed in the correct locations under cvt/data during conversion. But in order to help the downstream usage of the .p2o file by PatternEngine, they need to be relocated to the build/classes/data directory before being jarred up. If we stayed true to the profile setup, they should really be in build/classes/data/<profile>, but as we've seen this does not work in our imports (via script or build_db.xml). I don't know if there a library of profiles that could be searched, or some implicit way it could determine the profile to utilize.

In this multi vs merged scenario, its mutually exclusive nature means that the application jar will contain the converted code that can only work with one of the modes (either multi or merged). So there is no good reason to include the extra level of data/<profile>/ since only one profile can ever be included. You won't even have the .p2o files for the other profile so why add extra runtime nonsense here?

The bottom line: does the conversion work properly already? If so, then you can handle the non-standard move of your .p2o resources using build scripting instead of something built into FWD.

Greg Shah wrote:

The bottom line: does the conversion work properly already? If so, then you can handle the non-standard move of your .p2o resources using build scripting instead of something built into FWD.

I just completed a full build with my proposed update. I'll confirm with that, because my previous test I was just performing the jar on a build that was there. If the jar files are formatted the same, I think we are good with the build.xml changes.

I think we are good with handling this in the build.xml where applicable. The jar file contains the .p2o and .dict files in the data directory:

  9621938  2025-09-10 05:47   data/menu.p2o
  1597836  2025-09-10 05:47   data/records.p2o
   497567  2025-09-10 05:47   data/standard.p2o
 10848701  2025-09-10 05:47   data/menu.dict
  1907183  2025-09-10 05:47   data/records.dict
  2095041  2025-09-10 05:47   data/standard.dict

The build itself contains those files that are in the profile directory in the cvt/data/<profile> directory:

../../cvt/data/merged/menu.p2o
../../cvt/data/merged/menu.dict
../../cvt/data/merged/menu.schema
../../cvt/data/records.dict
../../cvt/data/records.schema
../../cvt/data/records.p2o
../../cvt/data/standard.dict
../../cvt/data/standard.p2o
../../cvt/data/standard.schema

The hotel samples do not have this configuration, but I can still setup the build.xml to contain the steps necessary to do so.

The hotel samples do not have this configuration, but I can still setup the build.xml to contain the steps necessary to do so.

Please don't. This is very esoteric. I don't want to carry around core build.xml stuff for one-off requirements.

Dănuț, did you test any configuration where you had 2 profiles, each one specifying a different fileset? If you could take a look at #9754-126 where I have in cfg/p2j.cfg.xml:

   <profile name="merged" default="true">
      <file-set filename="zfile_set.txt_merged" />
...
   </profile>

   <profile name="multi" default="false">
      <file-set filename="zfile_set.txt_multi" />
...
   </profile>

and in the build.xml, the convert target has <arg value="-P${profile}" /> option given to the ConversionDriver. It seems the multi fileset isn't being utilized when the multi profile is specified.

Roger Borrello wrote:

Dănuț, did you test any configuration where you had 2 profiles, each one specifying a different fileset? If you could take a look at #9754-126 where I have in cfg/p2j.cfg.xml:
[...]

and in the build.xml, the convert target has <arg value="-P${profile}" /> option given to the ConversionDriver. It seems the multi fileset isn't being utilized when the multi profile is specified.

The configuration you are referencing is not valid. The merged and multi profiles are mutually exclusive so you should NEVER try to run them at the same time.

Greg Shah wrote:

Roger Borrello wrote:

Dănuț, did you test any configuration where you had 2 profiles, each one specifying a different fileset? If you could take a look at #9754-126 where I have in cfg/p2j.cfg.xml:
[...]

and in the build.xml, the convert target has <arg value="-P${profile}" /> option given to the ConversionDriver. It seems the multi fileset isn't being utilized when the multi profile is specified.

The configuration you are referencing is not valid. The merged and multi profiles are mutually exclusive so you should NEVER try to run them at the same time.

Merry Christmas!

Not sure where you got the idea they are being run at the same time. Running a conversion with profile=merged (default) works, but running a conversion with profile=multi still tries to use the zfile_set.txt_merged

Roger Borrello wrote:

Greg Shah wrote:

Roger Borrello wrote:

Dănuț, did you test any configuration where you had 2 profiles, each one specifying a different fileset? If you could take a look at #9754-126 where I have in cfg/p2j.cfg.xml:
[...]

and in the build.xml, the convert target has <arg value="-P${profile}" /> option given to the ConversionDriver. It seems the multi fileset isn't being utilized when the multi profile is specified.

The configuration you are referencing is not valid. The merged and multi profiles are mutually exclusive so you should NEVER try to run them at the same time.

Merry Christmas!

Not sure where you got the idea they are being run at the same time. Running a conversion with profile=merged (default) works, but running a conversion with profile=multi still tries to use the zfile_set.txt_merged

Are you reporting a bug in FWD? Or is this a scripting issue for that application? The profile itself should define the file set.

Greg Shah wrote:

Are you reporting a bug in FWD? Or is this a scripting issue for that application? The profile itself should define the file set.

Is the profile able to specify a fileset "file" to use? If not, oops... that's a big difference the the fileset itself can only be specified. If it should be able to specify a file, then it is a bug.

As documented in Profiles (and discussed previously in #6256-127) you can specify the file-set using an inline approach or using the filename="<rogers_wonderful_file_name>" attribute. Check your file-set nodes to confirm that you have configured it properly.

Greg Shah wrote:

As documented in Profiles (and discussed previously in #6256-127) you can specify the file-set using an inline approach or using the filename="<rogers_wonderful_file_name>" attribute. Check your file-set nodes to confirm that you have configured it properly.

I am pretty sure the file-set node in p2j.cfg.xml is setup correctly. When the profile is multi, the output shows:

convert:
     [java] A profile was specified, previous set mode will be overridden.
     [java] Configuration profile set to 'multi'.
...

<?xml version="1.0"?>

<!-- P2J main configuration -->
<cfg>

   <global>
...
   </global>

   <profile name="merged" default="true">
      <file-set filename="zfile_set.txt_merged" />
      <schema>
         <namespace name="standard" />
...
      </schema>
   </profile>

   <profile name="multi" default="false">
      <file-set filename="zfile_set.txt_multi" />
      <schema>
         <namespace name="standard" />
...
      </schema>
   </profile>

   <schema>
...
   </schema>
</cfg>     

There's default=true which indicates that the profile is loaded by default, even if there is no explicit profile used. If you set "default" to false and try to run the commands, then there is no issue. The default profile will always be included.

Dănuț Filimon wrote:

There's default=true which indicates that the profile is loaded by default, even if there is no explicit profile used. If you set "default" to false and try to run the commands, then there is no issue. The default profile will always be included.

But if we do specify a profile, and the profile is "multi", why isn't the zfile_set.txt_multi fileset file used?

I tested by removing default="true" (I had already removed default="false", which seemed superfluous anyway) and it does indeed utilize the multi profile fileset. I do stand by my assumption that if we don't specify a profile, having a default is valid, and specifying a profile should override any default. I believe this is a bug.

Roger Borrello wrote:

I tested by removing default="true" (I had already removed default="false", which seemed superfluous anyway) and it does indeed utilize the multi profile fileset. I do stand by my assumption that if we don't specify a profile, having a default is valid, and specifying a profile should override any default. I believe this is a bug.

Oh, Happy New Year :-)

Roger Borrello wrote:

I tested by removing default="true" (I had already removed default="false", which seemed superfluous anyway) and it does indeed utilize the multi profile fileset. I do stand by my assumption that if we don't specify a profile, having a default is valid, and specifying a profile should override any default. I believe this is a bug.

It all depends on what we mean by default. If it means "always present" then we can't use it here because of the mutual exclusivity of merged and multi. If it means "use this if no other profile is specified" then you are right, it would be a bug.

I probably lean toward your interpretation. Constantin?

I went straight into work mode haha. Happy New Year!

Roger Borrello wrote:

I tested by removing default="true" (I had already removed default="false", which seemed superfluous anyway) and it does indeed utilize the multi profile fileset. I do stand by my assumption that if we don't specify a profile, having a default is valid, and specifying a profile should override any default. I believe this is a bug.

The profile is loaded initially in TransformDriver.processCommandLine() by

Configuration.loadConfigProfiles(cfgProfiles);

OK, fix it. It should behave like "use this if no other profile is specified".

Danut, you are right, the problem is here:

   private void loadConfigProfilesImpl(List<String> profiles)
   {
      if (profiles.isEmpty() || activeProfiles != null)
      {
         return;
      }

      if (profiles.size() == 1)
      {
         // <----------- when a single profile is used, the stuff bellow is not set, and the 'default' profile overrides it later on.
         loadProfile(profiles.get(0));
         return;
      }

      schemaConfig.deactiveDefaults();

      activeProfiles = new LinkedHashSet<>();

Greg Shah wrote:

OK, fix it. It should behave like "use this if no other profile is specified".

Can I put this task into WIP or should I create a separate one?

Dănuț Filimon wrote:

you can use the following in the meantime:

I'll await your commit.

Roger Borrello wrote:

Dănuț Filimon wrote:

you can use the following in the meantime:

I'll await your commit.

I posted this because I had trouble applying the patch. patch thought it was a reversal of an already applied patch. I looked at it, and it was simply removing some lines, so I applied it manually. But when I ran the conversion, there were issues with NPE:

     [java] com.goldencode.ast.AstException: Error processing ./abl/client/applhelp.p
     [java]     at com.goldencode.p2j.uast.AstGenerator.processFile(AstGenerator.java:1090)
     [java]     at com.goldencode.p2j.uast.ScanDriver.lambda$scan$0(ScanDriver.java:469)
     [java]     at com.goldencode.p2j.uast.ScanDriver.scan(ScanDriver.java:509)
     [java]     at com.goldencode.p2j.uast.ScanDriver.scan(ScanDriver.java:318)
     [java]     at com.goldencode.p2j.convert.TransformDriver.runScanDriver(TransformDriver.java:444)
     [java]     at com.goldencode.p2j.convert.TransformDriver.front(TransformDriver.java:300)
     [java]     at com.goldencode.p2j.convert.TransformDriver.executeJob(TransformDriver.java:1233)
     [java]     at com.goldencode.p2j.convert.ConversionDriver.main(ConversionDriver.java:1301)
     [java] Caused by: java.lang.NullPointerException: Cannot invoke "java.util.Map.get(Object)" because "com.goldencode.p2j.schema.SchemaDictionary.pristineDbs" is null
     [java]     at com.goldencode.p2j.schema.SchemaDictionary.loadSchema(SchemaDictionary.java:1262)
     [java]     at com.goldencode.p2j.schema.SchemaDictionary.loadSchema(SchemaDictionary.java:1238)
     [java]     at com.goldencode.p2j.uast.SymbolResolver.loadSchemaDatabases(SymbolResolver.java:6668)
     [java]     at com.goldencode.p2j.uast.AstGenerator.processParserHints(AstGenerator.java:1809)
     [java]     at com.goldencode.p2j.uast.AstGenerator.parse(AstGenerator.java:1642)
     [java]     at com.goldencode.p2j.uast.AstGenerator.processFile(AstGenerator.java:1078)
     [java]     ... 7 more

Roger Borrello wrote:

Roger Borrello wrote:

Dănuț Filimon wrote:

you can use the following in the meantime:

I'll await your commit.

I posted this because I had trouble applying the patch. patch thought it was a reversal of an already applied patch. I looked at it, and it was simply removing some lines, so I applied it manually. But when I ran the conversion, there were issues with NPE:
[...]
You might already know about it, so I'll back off until you post a commit.

I didn't find this issue, do you have a test case you can provide?

You have an activeProfile now, so the SchemaDictionary.pristineDbs will not be set in SchemaDictionary constructor. Something about having an active profile while only one profile is being used doesn't sound right.

I am setting up the scenario received from Roger right now.

Greg Shah wrote:

Code Review Task Branch 6256b Revision 16331

The change looks good to me.

Roger: Please test it.

It test's great! I have my cfg/p2j.cfg.xml setup with the default merged profile, and I can utilize that (the large fileset gets used), or override it and the smaller fileset gets used). I'm not sure there's much else to test on this function.

Roger Borrello wrote:

Greg Shah wrote:

Code Review Task Branch 6256b Revision 16331

The change looks good to me.

Roger: Please test it.

It test's great! I have my cfg/p2j.cfg.xml setup with the default merged profile, and I can utilize that (the large fileset gets used), or override it and the smaller fileset gets used). I'm not sure there's much else to test on this function.

Even if I don't set a default in cfg/p2j.cfg.xml, my build.xml sets the property by default, so there is a profile set in all my cases. I'd have to setup something special if we wanted to test an edge case, if you weren't able to test that yourself.

Testing went well... can this be merged to trunk soon?

Constantin can make the call.

I am rebasing 6256b right now.

I had a project that utilized various zfiles previously now needs to be ported to using profiles. There was a main/default profile (it was used earlier in this Redmine) named "merged" with an alternative named "multi". I'd like to incorporate 2 more, which were for development, both simply reduced the list of files that were included in the coversion, but associated with the merged schema.

Can I be pointed to the documentation that describes the p2j.cfg.xml? I was looking at Running the Front End Conversion but it only has one mention of that file in it. I was hoping I wouldn't just repeat the same merged section 3 times with different names, since the schema is exactly the same:

   <profile name="merged" default="true">
      <file-set filename="zfile_set.txt_merged" />
      <schema>
         <namespace name="standard" />

         <namespace
            name="records" 
            default="false" >
            <parameter name="ddl-dialects" value="postgresql" />
            <dialect-specific name="postgresql">
               <parameter name="udf" value="sql" />
            </dialect-specific>
         </namespace>
         <namespace
            name="menu" 
            importFile="data/menu.df" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
            <dialect-specific name="postgresql">
               <parameter name="udf" value="sql" />
            </dialect-specific>
         </namespace>
      </schema>
   </profile>

   <profile name="merged_menu_only">
      <file-set filename="zfile-set_build_menu_only.txt" />
      <schema>
         <namespace name="standard" />

         <namespace
            name="records" 
            default="false" >
            <parameter name="ddl-dialects" value="postgresql" />
            <dialect-specific name="postgresql">
               <parameter name="udf" value="sql" />
            </dialect-specific>
         </namespace>
         <namespace
            name="menu" 
            importFile="data/menu.df" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
            <dialect-specific name="postgresql">
               <parameter name="udf" value="sql" />
            </dialect-specific>
         </namespace>
      </schema>
   </profile>

   <profile name="merged_srv_only">
      <file-set filename="zfile-set_build_srv_only.txt" />
      <schema>
         <namespace name="standard" />

         <namespace
            name="records" 
            default="false" >
            <parameter name="ddl-dialects" value="postgresql" />
            <dialect-specific name="postgresql">
               <parameter name="udf" value="sql" />
            </dialect-specific>
         </namespace>
         <namespace
            name="menu" 
            importFile="data/menu.df" 
            default="true" >
            <parameter name="ddl-dialects" value="postgresql" />
            <dialect-specific name="postgresql">
               <parameter name="udf" value="sql" />
            </dialect-specific>
         </namespace>
      </schema>
   </profile>

I was just trying to determine if there was a different way to format so only the fileset and name would change.

See the Project Setup chapter in the Conversion Handbook.

Also, you can look at the Configuration class source code.