I18nOps.java
/*
** Module : I18nOps.java
** Abstract : Progress 4GL compatible I18N environment related methods.
**
** Copyright (c) 2019-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
** 001 EVL 20190112 Initial commit. Adding support for GET-CODEPAGES function. Adding runtime
** support for CODEPAGE-CONVERT function. Adding support for CHR/ASC functions.
** If source and target CP are the same - no need to do extra work. Adding
** implementation stubs fro CP* session attribute getters.
** GES 20190509 Moved code to the worker from the external version. The worker no longer uses
** BDT wrappers so it is more efficient if called from other runtime code.
** 002 ME 20200401 Make `UNDEFINED` property public, can't be used for longchar.
** 003 ME 20200604 Avoid NPE exception if text to convert is null.
** 004 RFB 20200722 Fixed javadoc issue
** 005 OM 20200918 Fixed UTF-8 variable-bytes support for chr(n, str, str).
** OM 20210330 Added data pre-validation which detects issues for which the Java Charsets are too
** permissible. Improved COPY-LOB engine and validations.
** OM 20210404 Fine-tuned errors generated by codepage management.
** OM 20210407 Reworked asc() and chr() functions.
** CA 20210423 Fixed I18nOps.convmap2Java access (key must be uppercased).
** OM 20210602 CPSTREAM defaults to the value configured in [p2j.cfg.xml] if not specified in directory.
** OM 20210907 CHR(0) returns the empty string. Fixed get4glCharacter() decoding for quad byte mode.
** 006 SBI 20220508 Changed getCharsetNative() to use Utils.getCharsetName().
** TJD 20220504 Java 11 compatibility minor changes
** 007 CA 20230110 Small improvement for context-local resolution.
** HC 20230118 Eliminated some of the uses of String.toUpperCase and/or String.toLowerCase
** for performance.
** 008 EVL 20230310 Implemented the set of codepages nedd the NULL byte to check for incoming binary data.
** Used now in prevalidateData() method.
** 009 RAA 20230525 Creating a new BaseDataType instance is now done using BaseDataTypeFactory.
** 010 TT 20230915 Read properties from StartupParameters.
** 011 OM 20231215 Implemented getCPColl / SESSION:CPCOLL to always return 'BASIC'.
** 012 SB 20240411 Added getJavaCPInternal to get the JVM charset for the CP defined in
** SESSION:CPINTERNAL.
** 013 SP 20240619 Added getWordDelimiters to get a regex list of delimiters based on codepage used.
** 014 SBI 20250227 Added missed javadoc for getJavaCharset(String) that can be used by the client code.
** 015 SP 20250314 Added getDelimiterPattern to retrieve a cached regex Pattern for word delimiters.
** 016 TG 20250516 Added getCodePage(BaseDataType) to retrieve codePage for CLOB and LONGCHAR.
** TG 20250519 Raising error when an instance is not a longchar or clob.
** TG 20250521 Reverting the previous change. 4GL does not throw error when is an other instance.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.util;
import java.io.*;
import java.nio.charset.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.*;
import com.goldencode.p2j.directory.*;
import com.goldencode.p2j.security.*;
import com.goldencode.util.*;
/**
* Progress compatible I18N-related environment static utility methods.
* <p>
* <pre>
* Progress Function Java Equivalent
* --------------------- ----------------------
* codepage-convert {@link #codePageConvert}
* get-codepages {@link #getCodePages}
* </pre>
* <p>
* <pre>
* System Handle:Attribute Java Equivalent
* ------------------------- ----------------------
* session:charset {@link #getCharset} (get)
* session:cpcase {@link #getCPCase} (get)
* session:cpcoll {@link #getCPColl} (get)
* session:cpinternal {@link #getCPInternal} (get)
* session:cplog {@link #getCPLog} (get)
* session:cpprint {@link #getCPPrint} (get)
* session:cprcodein {@link #getCPRCodeIn} (get)
* session:cprcodeout {@link #getCPRCodeOut} (get)
* session:cpstream {@link #getCPStream} (get)
* session:cpterm {@link #getCPTerm} (get)
* rcode-info:codepage {@link #getCodePage} (get)
* </pre>
*/
public class I18nOps
{
/** The 4GL name for ISO8859-1 / Latin-1 character encoding. */
public static final String ABL_LATIN1 = "ISO8859-1";
/** The Java name for ISO-8859-1 / Latin-1 character encoding. */
public static final String JAVA_LATIN1 = "ISO-8859-1";
/**
* Maps code pages from defined default convmap.cp to supported Java code page names.
*/
public static final Map<String, String> convmap2JavaDefault = new LinkedHashMap<>();
/**
* Maps code pages from currently defined default convmap.cp to supported Java code page names.
* This is the working copy after support verification for current JVM.
*/
public static final Map<String, String> convmap2Java = new CaseInsensitiveLinkedHashMap<>();
/**
* Maps code pages from supported Java code page names to Progress compatible code pages.
* This is the working copy after support verification for current JVM.
*/
public static final Map<String, String> convmap2Progress = new LinkedHashMap<>();
/** Special code page meaning no code page conversion for either source or target. */
public static final String CP_UNDEFINED = "UNDEFINED";
/** The standard name of the 4GL file containing the conversion map. */
public static final String CONVERSION_MAP_FILE = "convmap.cp";
/** Stores context-local state variables. */
private static final ContextContainer work = new ContextContainer();
/** Stores the current list of the code pages to do NULL byte check. */
private static final Set<String> zeroByteCheck;
/** Maps code pages to regex list of delimiters used for word splitting. */
private static final Map<String, String> cp2Delimiters = new HashMap<>();
/** Cache for regex patterns used in word splitting. */
private static final Map<String, Pattern> delimiterPatterns = new ConcurrentHashMap<>(8);
// initialize code page mappings to Java compatible names
static
{
// These mappings are explicitly being added in the same exact order
// they appear in the 4GL GET-CODEPAGES() function. Do not change
// the order, otherwise the result of that function will be incorrect.
//---------------------- 4GL CP, Java CP
convmap2JavaDefault.put("1256", "windows-1256");
convmap2JavaDefault.put("709", "709"); // support is undefined
convmap2JavaDefault.put("708", "708"); // support is undefined
convmap2JavaDefault.put("721", "721"); // support is undefined
convmap2JavaDefault.put("711", "711"); // support is undefined
convmap2JavaDefault.put("786", "786"); // support is undefined
convmap2JavaDefault.put("714", "714"); // support is undefined
convmap2JavaDefault.put("710", "710"); // support is undefined
convmap2JavaDefault.put("720", "720"); // support is undefined
convmap2JavaDefault.put("BIG-5", "Big5");
convmap2JavaDefault.put("GB2312", "GB2312");
convmap2JavaDefault.put("CP936", "CP936"); // support is undefined
convmap2JavaDefault.put("CP950", "CP950"); // support is undefined
convmap2JavaDefault.put("IBM852", "IBM852");
convmap2JavaDefault.put("1250", "windows-1250");
convmap2JavaDefault.put("ISO8859-2", "ISO-8859-2");
convmap2JavaDefault.put("1253", "windows-1253");
convmap2JavaDefault.put("IBM851", "IBM851"); // support is undefined
convmap2JavaDefault.put("ISO8859-8", "ISO-8859-8");
convmap2JavaDefault.put("IBM862", "IBM862");
convmap2JavaDefault.put("IBM850", "IBM850");
convmap2JavaDefault.put("IBM858", "IBM858"); // support is undefined
convmap2JavaDefault.put(ABL_LATIN1, JAVA_LATIN1);
convmap2JavaDefault.put("ISO8859-15", "ISO-8859-15");
convmap2JavaDefault.put("SHIFT-JIS", "Shift_JIS");
convmap2JavaDefault.put("EUCJIS", "EUCJIS"); // support is undefined
convmap2JavaDefault.put("KSC5601", "KSC5601"); // support is undefined
convmap2JavaDefault.put("CP949", "x-windows-949");
convmap2JavaDefault.put("CP1361", "CP1361"); // support is undefined
convmap2JavaDefault.put("1252", "windows-1252");
convmap2JavaDefault.put("1257", "windows-1257");
convmap2JavaDefault.put("MAZOVIA", "MAZOVIA"); // support is undefined
convmap2JavaDefault.put("ROMAN-8", "ROMAN-8"); // support is undefined
convmap2JavaDefault.put("KOI8-R", "KOI8-R");
convmap2JavaDefault.put("1251", "windows-1251");
convmap2JavaDefault.put("IBM866", "IBM866");
convmap2JavaDefault.put("ISO8859-5", "ISO-8859-5");
convmap2JavaDefault.put("620-2533", "x-iso-8859-11");
convmap2JavaDefault.put("1254", "windows-1254");
convmap2JavaDefault.put("IBM857", "IBM857");
convmap2JavaDefault.put(CP_UNDEFINED, CP_UNDEFINED);
convmap2JavaDefault.put("IBM861", "IBM861");
convmap2JavaDefault.put("IBM437", "IBM437");
convmap2JavaDefault.put("UTF-8", "UTF-8");
convmap2JavaDefault.put("UCS2", "UCS2"); // support is undefined
convmap2JavaDefault.put("UTF-32", "UTF-32");
convmap2JavaDefault.put("UTF-16", "UTF-16");
convmap2JavaDefault.put("UTF-16BE", "UTF-16BE");
convmap2JavaDefault.put("UTF-16LE", "UTF-16LE");
convmap2JavaDefault.put("UTF-32BE", "UTF-32BE");
convmap2JavaDefault.put("UTF-32LE", "UTF-32LE");
convmap2JavaDefault.put("ISO6937", "ISO6937"); // support is undefined
convmap2JavaDefault.put("CP950-HKSCS", "x-MS950-HKSCS");
convmap2JavaDefault.put("GB18030", "GB18030");
// possible list of the codepages to do NULL byte checking
String[] zeroByteCheckCandidates =
{"1250", "1251", "1252", "1253", "1254", "1256", "1257", "620-2533", "708", "709", "710", "711",
"714", "720", "721", "786", "IBM437", "IBM850", "IBM851", "IBM852", "IBM857", "IBM858", "IBM861",
"IBM862", "IBM866", "ISO8859-1", "ISO8859-2", "ISO8859-5", "ISO8859-8", "ISO8859-15", "KOI8-R",
"MAZOVIA", "ROMAN-8"};
Set<String> zeroByteCheckTmp = new HashSet<String>();
// set up the codepages to check for 0 byte
// undefined and latin1 should always be here
zeroByteCheckTmp.add(ABL_LATIN1);
zeroByteCheckTmp.add(CP_UNDEFINED);
for (String cpName : zeroByteCheckCandidates)
{
String cpJavaName = convmap2JavaDefault.get(cpName);
if (cpJavaName != null && Charset.isSupported(cpJavaName))
{
zeroByteCheckTmp.add(cpName);
}
}
// make unmodifiable list
zeroByteCheck = Collections.unmodifiableSet(zeroByteCheckTmp);
// check if the current Java environment supports all defaults
for (String csName : convmap2JavaDefault.keySet())
{
String csJavaName = convmap2JavaDefault.get(csName);
// special UNDEFINED code page is always supported
if (Charset.isSupported(csJavaName) || CP_UNDEFINED.equals(csJavaName))
{
// add supported code page to current convmap for further usage
convmap2Java.put(csName, csJavaName);
// back mapping to improve search performance
convmap2Progress.put(csJavaName, csName);
}
}
cp2Delimiters.put("ISO8859-1", "[\\x01-\\x22\\x26\\x28-\\x2f\\x3a-\\x3f\\x5b-\\x5e\\x60\\x7b-\\x89"
+ "\\x9f\\xa1-\\xa3\\xa5\\xa9\\xab\\xb8\\xbb\\xbc\\xbd\\xbf\\xc5\\xc6\\xd7\\xd8\\xe5\\xe6\\xf8]");
cp2Delimiters.put("ISO8859-15", "[\\x01-\\x22\\x26\\x28-\\x2f\\x3a-\\x3f\\x5b-\\x5e\\x60\\x7b-\\x83"
+ "\\x85-\\x87\\x89\\xa1\\xa3\\xab\\xbb\\xc5\\xc6\\xd8\\xe5\\xe6\\xf8]");
cp2Delimiters.put("1252", "[\\x01-\\x22\\x26\\x28-\\x2f\\x3a-\\x3f\\x5b-\\x5e\\x60\\x7b-\\x89"
+ "\\x8f\\xa1\\xa2\\xa5\\xa9\\xab\\xb8\\xbb-\\xbd\\xbf\\xc5\\xc6\\xd8\\xe5\\xe6\\xf8]");
cp2Delimiters.put("IBM850", "[\\x01-\\x22\\x26\\x28-\\x2f\\x3a-\\x3f\\x5b-\\x5e\\x60\\x7b-\\x7f\\x9c"
+ "\\x9e\\x9f\\xa6-\\xa9\\xab-\\xb4\\xb8-\\xc5\\xc8-\\xcf\\xd9-\\xdd\\xdf\\xe6\\xee-\\xff]");
cp2Delimiters.put("UTF-8", "[\\x01-\\x22\\x26\\x28-\\x2f\\x3a-\\x3f\\x5b-\\x5e\\x60\\x7b-\\x7f]");
}
/**
* Gets the comma separated list of the codepages supported by the conversion map for the
* Progress compatible environment.
* <p>
* In original environment the map is defined either in <code>convmap.cp</code> file or
* by the <code>-convmap</code> startup parameter for current ABL session.
*
* @return The list of the currently supported codepages.
*/
public static character getCodePages()
{
// TODO: implement me with taking into account the convmap.cp compatibility
StringBuilder sbRes = new StringBuilder();
// getting available code pages
Iterator<String> csIter = convmap2Java.keySet().iterator();
while (csIter.hasNext())
{
sbRes.append(csIter.next().toUpperCase());
if (csIter.hasNext())
{
sbRes.append(",");
}
}
return new character(sbRes.toString());
}
/**
* Returns the code page of the specified longchar/clob variable.
* If the variable is not a longchar/clob, the function will return an empty character.
*
* @param value
* The value for which the codepage will be retrieved.
*
* @return See above
*/
public static character getCodePage(BaseDataType value)
{
if (value instanceof longchar)
{
return ((longchar) value).getCodePage();
}
else if (value instanceof clob)
{
return ((clob) value).getCodePage();
}
return character.UNKNOWN;
}
/**
* Checks if the given code page name in currently defined convmap registry.
*
* @param cpName
* The name of the Progress code page name.
* Can be passed with lower case
*
* @return <code>TRUE</code> is code page is in current convmap,
* <code>FALSE</code> otherwise.
*/
public static boolean isSupported(String cpName)
{
return convmap2Java.containsKey(cpName);
}
/**
* Converts the given text using the specified codepage, assuming the text is encoded with
* the given source codepage.
*
* @param source
* The text to be converted.
* @param targetCP
* The target codepage.
* @param sourceCP
* The source codepage. If the text is a {@link longchar}
* instance, then the text's internal codepage is used.
*
* @return The converted text.
*/
public static <T extends Text> T codePageConvert(T source, String targetCP, String sourceCP)
{
if (source == null)
{
return null;
}
try
{
// create return value instance based on input parameter type
T resTxt = (T) BaseDataTypeFactory.instantiate(source.getClass());
// the return value has been constructed
if (source instanceof longchar)
{
((longchar) resTxt).fixCodePage(targetCP);
}
resTxt.assign(source.getValue());
return resTxt;
}
catch (ReflectiveOperationException roe)
{
// text instance is incompatible with extends Text
return null;
}
}
/**
* Converts the given text using the specified codepage, assuming the text is encoded with
* the given source codepage. This is the internal version to be called for service purposes.
*
* @param text
* The text to be converted.
* @param srcCP
* The source codepage.
* @param tarCP
* The target codepage.
*
* @return The converted text.
*/
@Deprecated
public static String codePageConvertWorker(String text, String srcCP, String tarCP)
{
String result = text;
String srcJCP = null;
String tarJCP = null;
// check if the target codepage was specified, no target then no conversion
if (tarCP != null && !tarCP.isEmpty())
{
// convert to uppercase version
tarJCP = convmap2Java.get(tarCP);
if (tarJCP != null)
{
// target CP is valid, check the source CP
if (srcCP == null || srcCP.isEmpty())
{
srcCP = _getCharset();
}
srcJCP = convmap2Java.get(srcCP);
// source/target CP validation completed
if (srcJCP != null)
{
// if same source and target CP's, don't do anything else
if (text != null && !srcJCP.equals(tarJCP))
{
// the real conversion is here
try
{
// undefined encoding forces the source to be untouched
byte[] bytes = CP_UNDEFINED.equals(srcJCP) ? text.getBytes() : text.getBytes(srcJCP);
// undefined encoding forces the target to be untouched
result = CP_UNDEFINED.equals(tarJCP) ? new String(bytes) : new String(bytes, tarJCP);
}
catch (UnsupportedEncodingException uee)
{
// TODO: does the 4GL really ignore this condition?
}
}
}
else
{
// TODO: no mapping for source CP name, log this or add the 4GL error
}
}
else
{
// TODO: no mapping for target CP name, log this or add the 4GL error
}
}
return result;
}
/**
* Converts the given integer character code to char value according to source and target
* codepage values.
*
* @param ascCode
* The key code to be converted.
* @param targetCP
* The target codepage of the char to return (Progress code page name).
* @param sourceCP
* The source codepage of the integer key code (Progress code page name).
* @param defaultCP
* The default codepage to be used when target or source codepages is invalid.
* This is the Java code page name which must be always valid.
*
* @return The converted single character value inside integer.
*/
public static int chr(int ascCode, String targetCP, String sourceCP, String defaultCP)
{
// default return value is invalid char
int intChRes = 0xFFFF;
// calculate Java codepages for source and target
String targetJavaCP = null;
String sourceJavaCP = null;
if (targetCP == null || targetCP.isEmpty())
{
targetJavaCP = CP_UNDEFINED;
}
else
{
targetJavaCP = convmap2Java.get(targetCP);
}
// if target code page is not valid we return invalid character
if (targetJavaCP != null)
{
// source code page was not specified, use default charset for session
if (sourceCP == null || sourceCP.isEmpty())
{
sourceJavaCP = defaultCP;
}
else
{
sourceJavaCP = convmap2Java.get(sourceCP);
}
// here we should have both target and source code pages validated
if (sourceJavaCP != null)
{
// if target is undefined, no conversion should occur
if (CP_UNDEFINED.equals(targetJavaCP) || sourceJavaCP.equals(targetJavaCP))
{
intChRes = ascCode;
}
else
{
// start conversion
try
{
// TODO: is this valid? Add source code page usage
// byte[] baSrc = ascCode >= 0 && ascCode <= 255
// ? new byte[] {(byte)ascCode}
// : new byte[] {(byte)(ascCode >> 8), (byte)ascCode};
// byte[] baSrc = new byte[] {(byte)ascCode, (byte)(ascCode >> 8),
// (byte)(ascCode >> 16), (byte)(ascCode >> 24)};
// String strSrc = CP_UNDEFINED.equals(sourceJavaCP)
// ? new String(baSrc) : new String(baSrc, sourceJavaCP);
// this approach ignores source code page option for out of the 255 codes
// considering unicode
String strSrc = null;
if (ascCode >= 0 && ascCode <= 255)
{
byte[] baSrc = new byte[] {(byte)ascCode};
strSrc = CP_UNDEFINED.equals(sourceJavaCP)
? new String(baSrc) : new String(baSrc, sourceJavaCP);
}
else
{
int[] codePointArr = new int[] {ascCode};
strSrc = new String(codePointArr, 0, 1);
}
byte [] baSrcTmp = strSrc.getBytes(targetJavaCP);
// undefined encoding forces the target to be untouched
String strRes = new String(baSrcTmp, targetJavaCP);
intChRes = strRes.codePointAt(0);
}
catch (UnsupportedEncodingException uee)
{
// nothing to do for now just leave return value as invalid
// DEBUGGING, remove when done
int dummy = 0;
}
}
}
}
return intChRes;
}
/**
* Converts the given integer character code to char value according to source and target
* codepage values (double byte enabled).
*
* @param ascCode
* The key code to be converted.
* @param targetCP
* The target codepage of the char to return (Progress code page name).
* @param sourceCP
* The source codepage of the integer key code (Progress code page name).
*
* @return The converted single character value inside integer.
*/
public static String chr(int ascCode, String targetCP, String sourceCP)
{
WorkArea wa = work.obtain();
// calculate Java codepages for source and target
if (sourceCP == null || sourceCP.equalsIgnoreCase(CP_UNDEFINED))
{
sourceCP = _getCPInternal(); // default to CPINTERNAL if not specified
}
if (targetCP == null || targetCP.equalsIgnoreCase(CP_UNDEFINED))
{
targetCP = _getCPInternal(); // default to CPINTERNAL if not specified
}
Charset sourceJavaCP = null;
Charset targetJavaCP = null;
try
{
sourceJavaCP = Charset.forName(convmap2Java.get(sourceCP));
targetJavaCP = Charset.forName(convmap2Java.get(targetCP));
}
catch (Exception ex)
{
// ignore errors
}
if (sourceJavaCP == null || targetJavaCP == null)
{
// Oddly enough 4GL first check if there is a mapping in convmap but only if source and target are
// different. If both source and target are the same invalid value then the error is 912, codepage
// not found.
if (sourceCP.equalsIgnoreCase(targetCP))
{
ErrorManager.recordOrThrowError(912, sourceCP, CONVERSION_MAP_FILE);
// Code page attribute table for <code-page> was not found in <filename>. (912)
}
else
{
final String msg1 = "Code page conversion table for %s to %s was not found in %s";
final String msg2 = "CHR could not find the appropriate conversion table in %s";
ErrorManager.recordOrThrowError(new int[] { 6063, 1586 },
new String[] {
String.format(msg1, sourceCP, targetCP, CONVERSION_MAP_FILE),
String.format(msg2, CONVERSION_MAP_FILE) }, false);
}
return null;
}
// interpret key code using source codepage
String chr = get4glCharacter(ascCode, sourceJavaCP);
// 4gl checks for lead byte, however there should be only one character
if (chr == null || chr.length() > 1)
{
return null;
}
return chr;
}
/**
* Use the specified {@code charset} to construct a string containing the code of character passed as
* first parameter.
*
* @param ascCode
* The code (double byte supported) of the character to be converted.
* @param charset
* The charset to be used in conversion.
*
* @return The string as noted above. If the {@code ascCode} is negative or {@code charset} is somehow
* unable to process the input, {@code null} is returned.
*/
private static String get4glCharacter(int ascCode, Charset charset)
{
// return unknown value for negative arguments
if (ascCode < 0)
{
return null;
}
// there is no "\0" character in 4GL. Returning empty string.
if (ascCode == 0)
{
return "";
}
byte[] data;
if (ascCode < 0x0100)
{
// single byte case
data = new byte[] { (byte) ascCode };
}
else if (ascCode < 0x010000)
{
// double byte mode, BigEndian mode
data = new byte[2];
data[0] = (byte) ((ascCode >>> 8) & 0x0FF);
data[1] = (byte) (ascCode & 0x0FF);
}
else if (ascCode < 0x01000000)
{
// triple byte mode, BigEndian mode
data = new byte[3];
data[0] = (byte) ((ascCode >>> 16) & 0x0FF);
data[1] = (byte) ((ascCode >>> 8) & 0x0FF);
data[2] = (byte) (ascCode & 0x0FF);
}
else
{
// quad byte mode, BigEndian mode
data = new byte[4];
data[0] = (byte) ((ascCode >>> 24) & 0x0FF);
data[1] = (byte) ((ascCode >>> 16) & 0x0FF);
data[2] = (byte) ((ascCode >>> 8) & 0x0FF);
data[3] = (byte) (ascCode & 0x0FF);
}
try
{
String ret = new String(data, charset);
if (ret.length() == 1)
{
char test = ret.charAt(0);
if (test == 0xFFFD || // REPLACEMENT CHARACTER
test == 0xFFFE || // not unassigned
test == 0xFFFF) // not unassigned
{
// debatable whether this is an error, from tests 4GL does not raise any condition, usually
return "";
}
}
return ret;
}
catch (Exception e)
{
// if the charset is unable to convert the data to a valid string
return null;
}
}
/**
* Converts the given char value to integer character code according to source and target
* codepage values.
*
* @param intChValue
* The character value to be converted packed into int.
* @param targetCP
* The target codepage of the integer key code to return (Progress code page name).
* @param sourceCP
* The source codepage of the char (Progress code page name).
* @param defaultCP
* The default codepage to be used when target or source codepages is invalid.
* This is the Java code page name which must be always valid.
*
* @return The converted single character value.
*/
public static int asc(int intChValue, String targetCP, String sourceCP, String defaultCP)
{
// default return value is invalid key code
int intRes = -1;
// calculate Java codepages for source and target
String targetJavaCP = null;
String sourceJavaCP = null;
// need to validate code pages first
if (targetCP == null || targetCP.isEmpty())
{
targetJavaCP = CP_UNDEFINED;
}
else
{
targetJavaCP = convmap2Java.get(targetCP);
}
// if target code page is not valid we return invalid character
if (targetJavaCP != null)
{
// source code page was not specified, use default charset for session
if (sourceCP == null || sourceCP.isEmpty())
{
sourceJavaCP = defaultCP;
}
else
{
sourceJavaCP = convmap2Java.get(sourceCP);
}
// here we should have both target and source code pages validated
if (sourceJavaCP != null)
{
// if target is undefined, no conversion should occur
if (CP_UNDEFINED.equals(targetJavaCP) || sourceJavaCP.equals(targetJavaCP))
{
intRes = intChValue;
}
else
{
// start conversion
try
{
byte[] baSrc = new byte[] {(byte)intChValue, (byte)(intChValue >> 8)};
String strSrc = CP_UNDEFINED.equals(sourceJavaCP)
? new String(baSrc) : new String(baSrc, sourceJavaCP);
byte [] baRes = strSrc.getBytes(targetJavaCP);
// reconstruct returning integer from array of bytes
intRes = 0;
for (int i = 0; i < baRes.length; i ++)
{
// this can be paranoid but ensure we do not exceed 32-bit integer
// 0 byte means stop reconstruction
if (i > 3 || baRes[i] == 0)
{
break;
}
// the approach is to push byte values until 0
intRes = (intRes << 8) | (0x000000FF & baRes[i]);
}
}
catch (UnsupportedEncodingException uee)
{
// nothing to do for now just leave return value as invalid DEBUGGING, remove when done
int dummy = 0;
}
}
}
}
return intRes;
}
/**
* Converts the given char value to integer character code according to source and target codepage values.
*
* @param chValue
* The character to convert back to char point.
* @param targetCP
* The target codepage, if set conversion from source codepage is done before (Progress code
* page name).
* @param sourceCP
* The source codepage, the char point is relative to this (Progress code page name).
*
* @return The converted single character value, -1 if not a single character or invalid codepages
* specified.
*/
public static int asc(String chValue, String targetCP, String sourceCP)
{
if (chValue == null || chValue.isEmpty())
{
return -1;
}
// calculate Java codepages for source and target
if (sourceCP == null || sourceCP.equalsIgnoreCase(CP_UNDEFINED))
{
sourceCP = _getCPInternal();
}
String sourceJavaCP = convmap2Java.get(sourceCP);
if (sourceJavaCP == null)
{
return raiseErrorAndReturn(912, sourceCP.toUpperCase(), CONVERSION_MAP_FILE, null);
// Code page attribute table for <code-page> was not found in <filename>. (912)
}
if (chValue.length() != 1)
{
return -1;
}
if (targetCP == null || targetCP.equalsIgnoreCase(CP_UNDEFINED))
{
targetCP = _getCPInternal();
}
String targetJavaCP = convmap2Java.get(targetCP);
if (targetJavaCP == null)
{
ErrorManager.recordOrShowError(6063, sourceCP.toUpperCase(), targetCP.toUpperCase(), CONVERSION_MAP_FILE);
// Code page conversion table for <table-name> to <table-name> was not found in <file-name>. (6063)
return raiseErrorAndReturn(1587, CONVERSION_MAP_FILE, "", null);
// ASC could not find the appropriate conversion table in <file-name>. (1587)
}
try
{
byte[] bytes = chValue.getBytes(targetJavaCP);
int ret = bytes[0] & 0xFF;
if (bytes.length > 1)
{
ret = (ret << 8) | (bytes[1] & 0xFF);
}
if (bytes.length > 2)
{
ret = (ret << 8) | (bytes[2] & 0xFF);
}
if (bytes.length > 3)
{
ret = (ret << 8) | (bytes[3] & 0xFF);
}
if (ret == 0x00EFBFBD)
{
return 0;
}
return ret;
}
catch (UnsupportedEncodingException e)
{
ErrorManager.recordOrShowError(6063, sourceCP.toUpperCase(), targetCP.toUpperCase(), CONVERSION_MAP_FILE);
// Code page conversion table for <table-name> to <table-name> was not found in <file-name>. (6063)
return -1;
}
}
/**
* This function WILL return value in NO-ERROR mode even if a fatal condition occurred.
*
* @param errCode
* The code of the error to be raised.
* @param p1
* The first parameter.
* @param p3
* The second parameter.
*
* @return Always -1 when executed in silent mode. Otherwise, will throw an exception and will NOT return.
*/
private static int raiseErrorAndReturn(int errCode, String p1, String p2, String p3)
{
if (ErrorManager.isSilentError())
{
ErrorManager.recordOrShowError(errCode, p1, p2);
return -1;
}
ErrorManager.recordOrThrowError(errCode, p1, p2);
return -2; // actually this point is never reached
}
/**
* Reports the current value of the CPINTERNAL attribute.
* <p>
* If the 4GL environment is actually configured for UTF-8, then this approach won't work.
* If the default codepage name for the JVM is not UTF-8, it is assumed that the codepage
* has been overriden already (at JVM startup). In that case, the value reported by the JVM
* is returned unchanged.
*
* @return The codepage used for internal operations.
*/
public static character getCPInternal()
{
return new character(_getCPInternal());
}
/**
* Reports the JVM charset that maps to the defined CPINTERNAL code page.
*
* @return The JVM corresponding charset or null.
*/
public static String getJavaCPInternal()
{
return convmap2Java.get(_getCPInternal());
}
/**
* Reports the current value of the CPINTERNAL attribute. String returning version.
* <p>
* If the 4GL environment is actually configured for UTF-8, then this approach won't work.
* If the default codepage name for the JVM is not UTF-8, it is assumed that the codepage
* has been overridden already (at JVM startup). In that case, the value reported by the JVM
* is returned unchanged.
*
* @return The codepage used for internal operations.
*/
public static String _getCPInternal()
{
WorkArea wa = work.get();
if (wa.cpInternal == null)
{
String cpInternal = SessionUtils._startupParameters().getCpInternal();
wa.cpInternal = cpInternal == null ? getCharsetNative() : cpInternal;
}
return wa.cpInternal;
}
/**
* Reports the default charset used by the JVM.
*
* @return The default charset name.
*/
public static character getCharset()
{
return new character(_getCharset());
}
/**
* Reports the default charset used by the JVM. String returning version.
*
* @return The default charset name.
*/
public static String _getCharset()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.charSet == null)
{
wa.charSet = _getCPInternal();
}
return wa.charSet;
}
/**
* Reports the default charset used by the JVM (Java mapping).
*
* @return The default charset name.
*/
public static String getJavaCharset()
{
return convmap2Java.get(_getCharset());
}
/**
* Reports the JVM charset that maps to the 4GL code page.
*
* @param proCharset
* The Progress code page
*
* @return The JVM corresponding charset or null.
*/
public static String getJavaCharset(character proCharset)
{
return getJavaCharset(proCharset.getValue());
}
/**
* Retrieve the java charset name corresponding to 4GL charset name. For example, if the given
* 4GL charset name is 1252, then the corresponding JVM charset name is WINDOWS-1252.
* This method is accessed on both the client and the server.
*
* @param proCharset
* The given 4GL charset name
*
* @return The corresponding java charset name.
*/
public static String getJavaCharset(String proCharset)
{
return proCharset == null ? null : convmap2Java.get(proCharset);
}
/**
* Reports the codepage used for streams (the equivalent of the CPSTREAM 4GL attribute), which
* reports the value set using the <code>-cpstream</code> startup parameter.
*
* @return The codepage used for streams.
*/
public static character getCPStream()
{
return new character(_getCPStream());
}
/**
* Reports the codepage used for streams (the equivalent of the CPSTREAM 4GL attribute, which
* reports the value set using the <code>-cpstream</code> startup parameter. String returning
* version.
*
* @return The codepage used for streams.
*/
public static String _getCPStream()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpStream == null)
{
wa.cpStream = _getCPInternal();
}
return wa.cpStream;
}
/**
* Gets the value of the SESSION:CPCASE attribute.
*
* @return The current CPCASE attribute for session.
*/
public static character getCPCase()
{
// for now this is the same as CPINTERNAL
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpCase == null)
{
wa.cpCase = _getCPInternal();
}
return new character(wa.cpCase);
}
/**
* Gets the value of the SESSION:CPCOLL attribute.
*
* @return The current CPCOLL attribute for session.
*/
public static character getCPColl()
{
WorkArea wa = work.obtain();
if (wa.cpColl == null)
{
wa.cpColl = "Basic"; // all string compare operations are done using the native Java String
}
return new character(wa.cpColl);
}
/**
* Gets the value of the SESSION:CPLOG attribute.
*
* @return The current CPLOG attribute for session.
*/
public static character getCPLog()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpLog == null)
{
wa.cpLog = _getCPInternal();
}
return new character(wa.cpLog);
}
/**
* Gets the value of the SESSION:CPPRINT attribute.
*
* @return The current CPPRINT attribute for session.
*/
public static character getCPPrint()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpPrint == null)
{
wa.cpPrint = _getCPInternal();
}
return new character(wa.cpPrint);
}
/**
* Gets the value of the SESSION:CPRCODEIN attribute.
*
* @return The current CPRCODEIN attribute for session.
*/
public static character getCPRCodeIn()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpRCodeIn == null)
{
wa.cpRCodeIn = _getCPInternal();
}
return new character(wa.cpRCodeIn);
}
/**
* Gets the value of the SESSION:CPRCODEOUT attribute.
*
* @return The current CPRCODEOUT attribute for session.
*/
public static character getCPRCodeOut()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpRCodeOut == null)
{
wa.cpRCodeOut = _getCPInternal();
}
return new character(wa.cpRCodeOut);
}
/**
* Gets the value of the SESSION:CPTERM attribute.
*
* @return The current CPTERM attribute for session.
*/
public static character getCPTerm()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.cpTerm == null)
{
wa.cpTerm = _getCPInternal();
}
return new character(wa.cpTerm);
}
/**
* Reports the codepage used for r-code file systems handle.
*
* @return The codepage used for r-code file.
*/
public static character getCodePage()
{
WorkArea wa = work.obtain();
// if override was not defined in directory.xml use current CPINTERNAL
if (wa.codePage == null)
{
wa.codePage = _getCPInternal();
}
return new character(wa.codePage);
}
/**
* Get word delimiter pattern.
*
* @param codePage
* codePage (character encoding) used.
*
* @return a regex pattern of word delimiters based on a codepage.
*/
public static Pattern getDelimiterPattern(String codePage)
{
return delimiterPatterns.computeIfAbsent(getWordDelimiters(codePage), Pattern::compile);
}
/**
* A regex list of word delimiters based on a codePage.
* If the delimiters for the specified {@code codePage} was not found,
* the delimiters for ISO8859-1 is returned as default.
*
* @param codePage
* codePage (character encoding) used.
*
* @return a regex list of word delimiters based on a codepage.
*/
public static String getWordDelimiters(String codePage)
{
if (codePage == null)
{
codePage = _getCPInternal();
}
String delimiters = cp2Delimiters.get(codePage.toUpperCase());
if (delimiters == null)
{
delimiters = cp2Delimiters.get("ISO8859-1");
}
return delimiters;
}
/**
* Analyze the first bytes of the {@code data} stream for BOM markers. If one is detected, the appropriate
* charset is ued to decode it and the resulting string is returned. If no BOM is detected, the default
* charset is used.
*
* @param data
* The data to be decoded.
* @param offset
* the offset where the returned string starts
* @param len
* The number of bytes to be processed. This might not be the actual size of the returned string
* because of BOM or multibyte character encoding.
*
* @return A string containing in {@code data} bytes, decoded automatically with detected charset.
*/
public static String getStringWithBomCheck(byte[] data, int offset, int len)
{
// check the BOM UTF-8: 0xEF_BB_BF
if (data.length >= 3 &&
(0xFF & data[offset]) == 0xEF &&
(0xFF & data[offset + 1]) == 0xBB &&
(0xFF & data[offset + 2]) == 0xBF)
{
return new String(data, offset + 3, len - 3, StandardCharsets.UTF_8);
}
// check the UTF-16: 0xFE_FF
else if (data.length >= 2 &&
(0xFF & data[offset]) == 0xEF &&
(0xFF & data[offset + 1]) == 0xFF)
{
return new String(data, offset + 2, len - 2, StandardCharsets.UTF_16);
}
// else : we have no information, take it as provided
else
{
String cpInternal = _getCPInternal();
if (!prevalidateData(data, offset, len, cpInternal))
{
ErrorManager.recordOrThrowError(12008, cpInternal.toUpperCase(), "");
// Invalid character code found in data for codepage (12008)
return null;
}
try
{
return new String(data, offset, len, convmap2Java.get(cpInternal));
}
catch (UnsupportedEncodingException e)
{
ErrorManager.recordOrThrowError(912, cpInternal, I18nOps.CONVERSION_MAP_FILE);
// Code page attribute table for <code-page> was not found in <filename>.
return null;
}
}
}
/**
* Prevalidate the byte array before allowing the Java charset to decode it to a string. We do this because
* the Java character decoder is more permissible than OE counterpart.
*
* @param data
* The byte array to be validated.
* @param offset
* the offset where the returned string starts
* @param len
* The number of bytes to be processed. This might not be the actual size of the returned string
* because of BOM or multibyte character encoding.
* @param oecp
* The original CP to be used in validation (only used for displaying OE errors).
* Not Java codepage.
*
* @return {@code true} if the data passed prevalidation. Otherwise {@code false} is returned.
*/
public static boolean prevalidateData(byte[] data, int offset, int len, String oecp)
{
// basic validation: in OE's single byte codepages, null characters ('\0') are not allowed
String cpUpCase = oecp != null && !oecp.isEmpty() ? oecp.toUpperCase() : CP_UNDEFINED;
if (zeroByteCheck.contains(cpUpCase))
{
for (int i = offset; i < len; i++)
{
if (data[i] == 0)
{
return false;
}
}
}
return true;
}
/**
* Initializes supplied class (aka referent class) for the purpose of i18n. The method returns a unique
* identifier associated with the supplied referent class. Any further i18n requests must contain this
* unique id.
* By convention a referent class has associated set of resource bundles, one for each target language.
* The resource bundles reside in the same Java package as the referent class, whereas the resource
* bundle class name is in the form [referent class simple name]_[target language].
* The current target language is determined by the assigned value to {@code CURRENT-LANGUAGE@} session
* variable.
*
* @param referentClass
* The referent class.
*
* @return Unique id of to the initialized referent class. Any further i18n requests must contain this
* id.
*/
public static long initReferent(Class<?> referentClass)
{
return TranslationManager.getInstance().initReferent(referentClass);
}
/**
* Reports the current value of the JVM charset properly mapped to valid 4GL code page.
* <p>
* Always return valid 4GL compatible charset or UNKNOWN one.
*
* @return The codepage used for internal operations as String.
*/
private static String getCharsetNative()
{
String javaCharset = Utils.getCharsetName();
// getting Progress name from respective map
String cpRes = convmap2Progress.get(javaCharset);
// this is practically impossible but to have NPE protection
if (cpRes == null || cpRes.isEmpty())
{
cpRes = CP_UNDEFINED;
}
return cpRes;
}
/**
* Helper to convert the character code from its integer value into string in Java native representation.
*
* @param charCode
* The integer to convert.
*
* @return The one-character {@code String} containing the character whose code was sent as parameter.
*/
private static String getNativeCharacter(int charCode, Charset charset)
{
byte[] bytes;
if (charset == StandardCharsets.UTF_8)
{
// see the table in https://en.wikipedia.org/wiki/UTF-8#Encoding for theoretical part
if (charCode <= 0x7F)
{
bytes = new byte[] {(byte) (charCode & 0b0_1111111) };
}
else if (charCode <= 0x7FF)
{
bytes = new byte[] {
(byte) (0b110_00000 | ((charCode >>> 6) & 0b000_11111)),
(byte) (0b10_000000 | ( charCode & 0b00_111111))
};
}
else if (charCode <= 0xFFFF)
{
bytes = new byte[] {
(byte) (0b1110_0000 | ((charCode >>> 12) & 0b0000_1111)),
(byte) (0b10_000000 | ((charCode >>> 6) & 0b00_111111)),
(byte) (0b10_000000 | ( charCode & 0b00_111111))
};
}
else if (charCode <= 0x10FFFF)
{
bytes = new byte[] {
(byte) (0b11110_000 | ((charCode >>> 18) & 0b00000_111)),
(byte) (0b10_000000 | ((charCode >>> 12) & 0b00_111111)),
(byte) (0b10_000000 | ((charCode >>> 6) & 0b00_111111)),
(byte) (0b10_000000 | ( charCode & 0b00_111111))
};
}
else
{
return ""; // not supported
}
}
// TODO: the UTF16 might also need variable byte encoding.
// NOTE: UTF16 comes in two flavours: are to endianness, see below
else if (charset == StandardCharsets.UTF_16 || charset == StandardCharsets.UTF_16BE)
{
// constant byte distribution
if ((charCode >>> 24) > 0)
{
// must check charset, this is a big-endian code, ex: StandardCharsets.UTF_16BE
bytes = new byte[]
{
(byte) (charCode >>> 24),
(byte) (charCode >>> 16),
(byte) (charCode >>> 8),
(byte) charCode
};
}
else if ((charCode >> 16) > 0)
{
bytes = new byte[]{(byte) (charCode >>> 16), (byte) (charCode >>> 8), (byte) charCode};
}
else if ((charCode >> 8) > 0)
{
bytes = new byte[]{(byte) (charCode >>> 8), (byte) charCode};
}
else
{
bytes = new byte[]{(byte) charCode};
}
}
else if (charCode < 0x100) // 7 or 8 bit encoding
{
if (charCode > 0x7F)
{
charCode -= 0x100;
}
bytes = new byte[] { (byte) charCode };
}
else
{
throw new RuntimeException("Do not know how to get the character " + charCode + " using " + charset);
}
return new String(bytes, charset);
}
/**
* Stores global data relating to the state of the current context.
*/
private static class WorkArea
{
/** The local or remote directory service to be used on both client and server sides. */
private final Directory dir = DirectoryManager.getInstance();
/** Stores any context-specific effective value for the <code>CPINTERNAL</code>. */
private String cpInternal = null;
/** Stores any context-specific effective value for the <code>CPSTREAM</code>. */
private String cpStream = SessionUtils._startupParameters().getCpStream();
/** Stores any context-specific effective value for the <code>CPCASE</code>. */
private String cpCase = SessionUtils._startupParameters().getCpCase();
/** Stores any context-specific effective value for the <code>CPCOLL</code>. */
private String cpColl = SessionUtils._startupParameters().getCpColl();
/** Stores any context-specific effective value for the <code>CPLOG</code>. */
private String cpLog = SessionUtils._startupParameters().getCpLog();
/** Stores any context-specific effective value for the <code>CPPRINT</code>. */
private String cpPrint = SessionUtils._startupParameters().getCpPrint();
/** Stores any context-specific effective value for the <code>CPRCODEIN</code>. */
private String cpRCodeIn = SessionUtils._startupParameters().getCpRCodeIn();
/** Stores any context-specific effective value for the <code>CPRCODEOUT</code>. */
private String cpRCodeOut = SessionUtils._startupParameters().getCpRCodeOut();
/** Stores any context-specific effective value for the <code>CPTERM</code>. */
private String cpTerm = SessionUtils._startupParameters().getCpTerm();
/** Stores any context-specific effective value for the <code>CHARSET</code>. */
private String charSet = SessionUtils._startupParameters().getCharset();
/** Stores any context-specific effective value for the <code>RCODE-INFO:CODEPAGE</code>. */
private String codePage = getOverrideFromDirectory("i18n/codepage");
/**
* Requests the server for code page related option available to be overridden in
* directory.xml file. All CP options can be overridden.
*
* @return The codepage related override from directory or <code>null</code> if none.
*/
private String getOverrideFromDirectory(String optName)
{
return dir.getString(Directory.ID_RELATIVE_BOTH, optName, null);
}
}
/**
* Simple container that stores and returns a context-local instance of
* the global work area.
*/
private static class ContextContainer
extends ContextLocal<WorkArea>
{
/**
* Obtains the context-local instance of the contained global work
* area.
*
* @return The work area associated with this context.
*/
public WorkArea obtain()
{
return this.get();
}
/**
* Initializes the work area, the first time it is requested within a
* new context.
*
* @return The newly instantiated work area.
*/
protected synchronized WorkArea initialValue()
{
WorkArea wa = new WorkArea();
return wa;
}
}
}