3871_1.patch
src/com/goldencode/p2j/schema/ImportWorker.java 2019-10-09 12:42:28 +0000 | ||
---|---|---|
139 | 139 |
** 050 CA 20190905 The data file is not mapped to memory, but to a byte buffer. This |
140 | 140 |
** allows very large dump files to be imported, without limitations from |
141 | 141 |
** the physical memory. |
142 |
** 051 SBI 20191008 Added the source converter for one byte encodings, set the source |
|
143 |
** encoding. |
|
142 | 144 |
*/ |
143 | 145 |
/* |
144 | 146 |
** This program is free software: you can redistribute it and/or modify |
... | ... | |
198 | 200 |
import java.io.*; |
199 | 201 |
import java.lang.InstantiationException; |
200 | 202 |
import java.lang.reflect.*; |
203 |
import java.nio.charset.*; |
|
201 | 204 |
import java.sql.*; |
202 | 205 |
import java.util.*; |
203 | 206 |
import java.util.logging.*; |
207 | ||
208 |
import com.goldencode.p2j.util.*; |
|
204 | 209 |
import com.goldencode.p2j.util.ErrorManager; |
210 | ||
205 | 211 |
import org.hibernate.*; |
206 | 212 |
import org.hibernate.cfg.*; |
207 | 213 |
import org.hibernate.dialect.*; |
208 | 214 |
import org.hibernate.service.*; |
209 | 215 |
import org.hibernate.type.*; |
210 | 216 |
import org.hibernate.type.Type; |
217 | ||
211 | 218 |
import com.goldencode.p2j.pattern.*; |
212 | 219 |
import com.goldencode.p2j.persist.*; |
213 | 220 |
import com.goldencode.p2j.persist.type.*; |
... | ... | |
1026 | 1033 |
|
1027 | 1034 |
// Open a new session and begin a transaction. |
1028 | 1035 |
session = openSession(); |
1036 |
|
|
1029 | 1037 |
Transaction tx = session.beginTransaction(); |
1030 | 1038 |
|
1031 | 1039 |
// Read up to batchSize records from input file. |
... | ... | |
2904 | 2912 |
/** The date format as it was set when the table was dumped. */ |
2905 | 2913 |
private String dateFormat = null; |
2906 | 2914 |
|
2915 |
/** The import source charset converter */ |
|
2916 |
private CharsetConverter charsetConverter; |
|
2917 |
|
|
2907 | 2918 |
/** |
2908 | 2919 |
* Constructor. |
2909 | 2920 |
* |
... | ... | |
2934 | 2945 |
// If any IO issue would occur, it should have happened in the super c'tor. |
2935 | 2946 |
ErrorManager.recordOrThrowError(98, "Unable to open file:" + filename + "."); |
2936 | 2947 |
} |
2948 |
|
|
2949 |
if (isSourceConversionRequired()) |
|
2950 |
{ |
|
2951 |
charsetConverter = createSourceCharsetConverter(); |
|
2952 |
} |
|
2953 |
else |
|
2954 |
{ |
|
2955 |
charsetConverter = null; |
|
2956 |
} |
|
2937 | 2957 |
} |
2938 | 2958 |
|
2939 | 2959 |
/** |
... | ... | |
2980 | 3000 |
} |
2981 | 3001 |
|
2982 | 3002 |
/** |
3003 |
* Returns the charset converter. |
|
3004 |
* |
|
3005 |
* @return The charset converter |
|
3006 |
*/ |
|
3007 |
protected CharsetConverter getCharsetConverter() |
|
3008 |
{ |
|
3009 |
if (charsetConverter != null) |
|
3010 |
{ |
|
3011 |
return charsetConverter; |
|
3012 |
} |
|
3013 |
|
|
3014 |
return super.getCharsetConverter(); |
|
3015 |
} |
|
3016 | ||
3017 |
/** |
|
2983 | 3018 |
* Reads the PSC footer, storing the key/values pairs in private map pscHeader. |
2984 | 3019 |
* |
2985 | 3020 |
* @return The number of PSC records actually read. If negative, the footer could not |
... | ... | |
3082 | 3117 |
encoding = getMetadata("cpstream"); // eg. ISO8859-15 |
3083 | 3118 |
// cc = new CharsetConverter(encoding); |
3084 | 3119 |
|
3120 |
setConvertSource(encoding); |
|
3121 |
|
|
3085 | 3122 |
ldbname = getMetadata("ldbname"); // eg. p2j_test |
3086 | 3123 |
timestamp = getMetadata("timestamp"); // eg. 2013/06/07-09:57:02 |
3087 | 3124 |
|
src/com/goldencode/p2j/util/FileStream.java 2019-10-09 12:31:10 +0000 | ||
---|---|---|
37 | 37 |
** 019 ECF 20171026 Added write(byte[], int, int) method. |
38 | 38 |
** 020 EVL 20180620 Adding pulse on close for empty frames. |
39 | 39 |
** 021 CA 20190905 Allow byte buffer instead of memory buffers for read-only files. |
40 |
** 022 SBI 20191008 Added getCharsetConverter(), createSourceCharsetConverter() and |
|
41 |
** isSourceConversionRequired(). |
|
40 | 42 |
*/ |
41 | 43 | |
42 | 44 |
/* |
... | ... | |
97 | 99 |
import java.io.*; |
98 | 100 |
import java.nio.*; |
99 | 101 |
import java.nio.channels.*; |
102 |
import java.nio.charset.*; |
|
100 | 103 | |
101 | 104 |
/** |
102 | 105 |
* A stream class supporting input and output semantics for any file-like |
... | ... | |
857 | 860 |
} |
858 | 861 |
|
859 | 862 |
/** |
863 |
* Returns the charset converter. |
|
864 |
* |
|
865 |
* @return The charset converter |
|
866 |
*/ |
|
867 |
protected CharsetConverter getCharsetConverter() |
|
868 |
{ |
|
869 |
return cc; |
|
870 |
} |
|
871 |
|
|
872 |
/** |
|
873 |
* Creates the source charset converter if the source code page is one byte encoding, otherwise |
|
874 |
* returns null value. |
|
875 |
* |
|
876 |
* @return The source charset converter |
|
877 |
*/ |
|
878 |
protected CharsetConverter createSourceCharsetConverter() |
|
879 |
{ |
|
880 |
CharsetConverter charsetConverter; |
|
881 |
|
|
882 |
try |
|
883 |
{ |
|
884 |
Charset charset = Charset.forName(sourceCp); |
|
885 |
|
|
886 |
if (charset.newEncoder().maxBytesPerChar() == 1) |
|
887 |
{ |
|
888 |
charsetConverter = new CharsetConverter(sourceCp); |
|
889 |
} |
|
890 |
else |
|
891 |
{ |
|
892 |
charsetConverter = null; |
|
893 |
} |
|
894 |
} |
|
895 |
catch(IllegalArgumentException | UnsupportedOperationException e) |
|
896 |
{ |
|
897 |
charsetConverter = null; |
|
898 |
} |
|
899 | ||
900 |
return charsetConverter; |
|
901 |
} |
|
902 |
|
|
903 |
/** |
|
904 |
* Tests if the source conversion is required such that if the code page is not equal to |
|
905 |
* the default code page. |
|
906 |
* |
|
907 |
* @return true if the source conversion is required, otherwise false. |
|
908 |
*/ |
|
909 |
protected boolean isSourceConversionRequired() |
|
910 |
{ |
|
911 |
Charset cs = Charset.forName(sourceCp); |
|
912 |
|
|
913 |
return !Charset.defaultCharset().equals(cs); |
|
914 |
} |
|
915 |
|
|
916 |
/** |
|
860 | 917 |
* Write a byte to the buffer, flushing if the buffer is full. |
861 | 918 |
* |
862 | 919 |
* @param b |
... | ... | |
961 | 1018 |
|
962 | 1019 |
if (convert && ch >= 0) |
963 | 1020 |
{ |
964 |
ch = cc.toChar(ch);
|
|
1021 |
ch = getCharsetConverter().toChar(ch);
|
|
965 | 1022 |
} |
966 | 1023 |
|
967 | 1024 |
return ch; |