/src/icu/source/common/ucnvmbcs.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // © 2016 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | /* |
4 | | ****************************************************************************** |
5 | | * |
6 | | * Copyright (C) 2000-2016, International Business Machines |
7 | | * Corporation and others. All Rights Reserved. |
8 | | * |
9 | | ****************************************************************************** |
10 | | * file name: ucnvmbcs.cpp |
11 | | * encoding: UTF-8 |
12 | | * tab size: 8 (not used) |
13 | | * indentation:4 |
14 | | * |
15 | | * created on: 2000jul03 |
16 | | * created by: Markus W. Scherer |
17 | | * |
18 | | * The current code in this file replaces the previous implementation |
19 | | * of conversion code from multi-byte codepages to Unicode and back. |
20 | | * This implementation supports the following: |
21 | | * - legacy variable-length codepages with up to 4 bytes per character |
22 | | * - all Unicode code points (up to 0x10ffff) |
23 | | * - efficient distinction of unassigned vs. illegal byte sequences |
24 | | * - it is possible in fromUnicode() to directly deal with simple |
25 | | * stateful encodings (used for EBCDIC_STATEFUL) |
26 | | * - it is possible to convert Unicode code points |
27 | | * to a single zero byte (but not as a fallback except for SBCS) |
28 | | * |
29 | | * Remaining limitations in fromUnicode: |
30 | | * - byte sequences must not have leading zero bytes |
31 | | * - except for SBCS codepages: no fallback mapping from Unicode to a zero byte |
32 | | * - limitation to up to 4 bytes per character |
33 | | * |
34 | | * ICU 2.8 (late 2003) adds a secondary data structure which lifts some of these |
35 | | * limitations and adds m:n character mappings and other features. |
36 | | * See ucnv_ext.h for details. |
37 | | * |
38 | | * Change history: |
39 | | * |
40 | | * 5/6/2001 Ram Moved MBCS_SINGLE_RESULT_FROM_U,MBCS_STAGE_2_FROM_U, |
41 | | * MBCS_VALUE_2_FROM_STAGE_2, MBCS_VALUE_4_FROM_STAGE_2 |
42 | | * macros to ucnvmbcs.h file |
43 | | */ |
44 | | |
45 | | #include "unicode/utypes.h" |
46 | | |
47 | | #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION |
48 | | |
49 | | #include "unicode/ucnv.h" |
50 | | #include "unicode/ucnv_cb.h" |
51 | | #include "unicode/udata.h" |
52 | | #include "unicode/uset.h" |
53 | | #include "unicode/utf8.h" |
54 | | #include "unicode/utf16.h" |
55 | | #include "ucnv_bld.h" |
56 | | #include "ucnvmbcs.h" |
57 | | #include "ucnv_ext.h" |
58 | | #include "ucnv_cnv.h" |
59 | | #include "cmemory.h" |
60 | | #include "cstring.h" |
61 | | #include "umutex.h" |
62 | | #include "ustr_imp.h" |
63 | | |
64 | | /* control optimizations according to the platform */ |
65 | | #define MBCS_UNROLL_SINGLE_TO_BMP 1 |
66 | | #define MBCS_UNROLL_SINGLE_FROM_BMP 0 |
67 | | |
68 | | /* |
69 | | * _MBCSHeader versions 5.3 & 4.3 |
70 | | * (Note that the _MBCSHeader version is in addition to the converter formatVersion.) |
71 | | * |
72 | | * This version is optional. Version 5 is used for incompatible data format changes. |
73 | | * makeconv will continue to generate version 4 files if possible. |
74 | | * |
75 | | * Changes from version 4: |
76 | | * |
77 | | * The main difference is an additional _MBCSHeader field with |
78 | | * - the length (number of uint32_t) of the _MBCSHeader |
79 | | * - flags for further incompatible data format changes |
80 | | * - flags for further, backward compatible data format changes |
81 | | * |
82 | | * The MBCS_OPT_FROM_U flag indicates that most of the fromUnicode data is omitted from |
83 | | * the file and needs to be reconstituted at load time. |
84 | | * This requires a utf8Friendly format with an additional mbcsIndex table for fast |
85 | | * (and UTF-8-friendly) fromUnicode conversion for Unicode code points up to maxFastUChar. |
86 | | * (For details about these structures see below, and see ucnvmbcs.h.) |
87 | | * |
88 | | * utf8Friendly also implies that the fromUnicode mappings are stored in ascending order |
89 | | * of the Unicode code points. (This requires that the .ucm file has the |0 etc. |
90 | | * precision markers for all mappings.) |
91 | | * |
92 | | * All fallbacks have been moved to the extension table, leaving only roundtrips in the |
93 | | * omitted data that can be reconstituted from the toUnicode data. |
94 | | * |
95 | | * Of the stage 2 table, the part corresponding to maxFastUChar and below is omitted. |
96 | | * With only roundtrip mappings in the base fromUnicode data, this part is fully |
97 | | * redundant with the mbcsIndex and will be reconstituted from that (also using the |
98 | | * stage 1 table which contains the information about how stage 2 was compacted). |
99 | | * |
100 | | * The rest of the stage 2 table, the part for code points above maxFastUChar, |
101 | | * is stored in the file and will be appended to the reconstituted part. |
102 | | * |
103 | | * The entire fromUBytes array is omitted from the file and will be reconstitued. |
104 | | * This is done by enumerating all toUnicode roundtrip mappings, performing |
105 | | * each mapping (using the stage 1 and reconstituted stage 2 tables) and |
106 | | * writing instead of reading the byte values. |
107 | | * |
108 | | * _MBCSHeader version 4.3 |
109 | | * |
110 | | * Change from version 4.2: |
111 | | * - Optional utf8Friendly data structures, with 64-entry stage 3 block |
112 | | * allocation for parts of the BMP, and an additional mbcsIndex in non-SBCS |
113 | | * files which can be used instead of stages 1 & 2. |
114 | | * Faster lookups for roundtrips from most commonly used characters, |
115 | | * and lookups from UTF-8 byte sequences with a natural bit distribution. |
116 | | * See ucnvmbcs.h for more details. |
117 | | * |
118 | | * Change from version 4.1: |
119 | | * - Added an optional extension table structure at the end of the .cnv file. |
120 | | * It is present if the upper bits of the header flags field contains a non-zero |
121 | | * byte offset to it. |
122 | | * Files that contain only a conversion table and no base table |
123 | | * use the special outputType MBCS_OUTPUT_EXT_ONLY. |
124 | | * These contain the base table name between the MBCS header and the extension |
125 | | * data. |
126 | | * |
127 | | * Change from version 4.0: |
128 | | * - Replace header.reserved with header.fromUBytesLength so that all |
129 | | * fields in the data have length. |
130 | | * |
131 | | * Changes from version 3 (for performance improvements): |
132 | | * - new bit distribution for state table entries |
133 | | * - reordered action codes |
134 | | * - new data structure for single-byte fromUnicode |
135 | | * + stage 2 only contains indexes |
136 | | * + stage 3 stores 16 bits per character with classification bits 15..8 |
137 | | * - no multiplier for stage 1 entries |
138 | | * - stage 2 for non-single-byte codepages contains the index and the flags in |
139 | | * one 32-bit value |
140 | | * - 2-byte and 4-byte fromUnicode results are stored directly as 16/32-bit integers |
141 | | * |
142 | | * For more details about old versions of the MBCS data structure, see |
143 | | * the corresponding versions of this file. |
144 | | * |
145 | | * Converting stateless codepage data ---------------------------------------*** |
146 | | * (or codepage data with simple states) to Unicode. |
147 | | * |
148 | | * Data structure and algorithm for converting from complex legacy codepages |
149 | | * to Unicode. (Designed before 2000-may-22.) |
150 | | * |
151 | | * The basic idea is that the structure of legacy codepages can be described |
152 | | * with state tables. |
153 | | * When reading a byte stream, each input byte causes a state transition. |
154 | | * Some transitions result in the output of a code point, some result in |
155 | | * "unassigned" or "illegal" output. |
156 | | * This is used here for character conversion. |
157 | | * |
158 | | * The data structure begins with a state table consisting of a row |
159 | | * per state, with 256 entries (columns) per row for each possible input |
160 | | * byte value. |
161 | | * Each entry is 32 bits wide, with two formats distinguished by |
162 | | * the sign bit (bit 31): |
163 | | * |
164 | | * One format for transitional entries (bit 31 not set) for non-final bytes, and |
165 | | * one format for final entries (bit 31 set). |
166 | | * Both formats contain the number of the next state in the same bit |
167 | | * positions. |
168 | | * State 0 is the initial state. |
169 | | * |
170 | | * Most of the time, the offset values of subsequent states are added |
171 | | * up to a scalar value. This value will eventually be the index of |
172 | | * the Unicode code point in a table that follows the state table. |
173 | | * The effect is that the code points for final state table rows |
174 | | * are contiguous. The code points of final state rows follow each other |
175 | | * in the order of the references to those final states by previous |
176 | | * states, etc. |
177 | | * |
178 | | * For some terminal states, the offset is itself the output Unicode |
179 | | * code point (16 bits for a BMP code point or 20 bits for a supplementary |
180 | | * code point (stored as code point minus 0x10000 so that 20 bits are enough). |
181 | | * For others, the code point in the Unicode table is stored with either |
182 | | * one or two code units: one for BMP code points, two for a pair of |
183 | | * surrogates. |
184 | | * All code points for a final state entry take up the same number of code |
185 | | * units, regardless of whether they all actually _use_ the same number |
186 | | * of code units. This is necessary for simple array access. |
187 | | * |
188 | | * An additional feature comes in with what in ICU is called "fallback" |
189 | | * mappings: |
190 | | * |
191 | | * In addition to round-trippable, precise, 1:1 mappings, there are often |
192 | | * mappings defined between similar, though not the same, characters. |
193 | | * Typically, such mappings occur only in fromUnicode mapping tables because |
194 | | * Unicode has a superset repertoire of most other codepages. However, it |
195 | | * is possible to provide such mappings in the toUnicode tables, too. |
196 | | * In this case, the fallback mappings are partly integrated into the |
197 | | * general state tables because the structure of the encoding includes their |
198 | | * byte sequences. |
199 | | * For final entries in an initial state, fallback mappings are stored in |
200 | | * the entry itself like with roundtrip mappings. |
201 | | * For other final entries, they are stored in the code units table if |
202 | | * the entry is for a pair of code units. |
203 | | * For single-unit results in the code units table, there is no space to |
204 | | * alternatively hold a fallback mapping; in this case, the code unit |
205 | | * is stored as U+fffe (unassigned), and the fallback mapping needs to |
206 | | * be looked up by the scalar offset value in a separate table. |
207 | | * |
208 | | * "Unassigned" state entries really mean "structurally unassigned", |
209 | | * i.e., such a byte sequence will never have a mapping result. |
210 | | * |
211 | | * The interpretation of the bits in each entry is as follows: |
212 | | * |
213 | | * Bit 31 not set, not a terminal entry ("transitional"): |
214 | | * 30..24 next state |
215 | | * 23..0 offset delta, to be added up |
216 | | * |
217 | | * Bit 31 set, terminal ("final") entry: |
218 | | * 30..24 next state (regardless of action code) |
219 | | * 23..20 action code: |
220 | | * action codes 0 and 1 result in precise-mapping Unicode code points |
221 | | * 0 valid byte sequence |
222 | | * 19..16 not used, 0 |
223 | | * 15..0 16-bit Unicode BMP code point |
224 | | * never U+fffe or U+ffff |
225 | | * 1 valid byte sequence |
226 | | * 19..0 20-bit Unicode supplementary code point |
227 | | * never U+fffe or U+ffff |
228 | | * |
229 | | * action codes 2 and 3 result in fallback (unidirectional-mapping) Unicode code points |
230 | | * 2 valid byte sequence (fallback) |
231 | | * 19..16 not used, 0 |
232 | | * 15..0 16-bit Unicode BMP code point as fallback result |
233 | | * 3 valid byte sequence (fallback) |
234 | | * 19..0 20-bit Unicode supplementary code point as fallback result |
235 | | * |
236 | | * action codes 4 and 5 may result in roundtrip/fallback/unassigned/illegal results |
237 | | * depending on the code units they result in |
238 | | * 4 valid byte sequence |
239 | | * 19..9 not used, 0 |
240 | | * 8..0 final offset delta |
241 | | * pointing to one 16-bit code unit which may be |
242 | | * fffe unassigned -- look for a fallback for this offset |
243 | | * ffff illegal |
244 | | * 5 valid byte sequence |
245 | | * 19..9 not used, 0 |
246 | | * 8..0 final offset delta |
247 | | * pointing to two 16-bit code units |
248 | | * (typically UTF-16 surrogates) |
249 | | * the result depends on the first code unit as follows: |
250 | | * 0000..d7ff roundtrip BMP code point (1st alone) |
251 | | * d800..dbff roundtrip surrogate pair (1st, 2nd) |
252 | | * dc00..dfff fallback surrogate pair (1st-400, 2nd) |
253 | | * e000 roundtrip BMP code point (2nd alone) |
254 | | * e001 fallback BMP code point (2nd alone) |
255 | | * fffe unassigned |
256 | | * ffff illegal |
257 | | * (the final offset deltas are at most 255 * 2, |
258 | | * times 2 because of storing code unit pairs) |
259 | | * |
260 | | * 6 unassigned byte sequence |
261 | | * 19..16 not used, 0 |
262 | | * 15..0 16-bit Unicode BMP code point U+fffe (new with version 2) |
263 | | * this does not contain a final offset delta because the main |
264 | | * purpose of this action code is to save scalar offset values; |
265 | | * therefore, fallback values cannot be assigned to byte |
266 | | * sequences that result in this action code |
267 | | * 7 illegal byte sequence |
268 | | * 19..16 not used, 0 |
269 | | * 15..0 16-bit Unicode BMP code point U+ffff (new with version 2) |
270 | | * 8 state change only |
271 | | * 19..0 not used, 0 |
272 | | * useful for state changes in simple stateful encodings, |
273 | | * at Shift-In/Shift-Out codes |
274 | | * |
275 | | * |
276 | | * 9..15 reserved for future use |
277 | | * current implementations will only perform a state change |
278 | | * and ignore bits 19..0 |
279 | | * |
280 | | * An encoding with contiguous ranges of unassigned byte sequences, like |
281 | | * Shift-JIS and especially EUC-TW, can be stored efficiently by having |
282 | | * at least two states for the trail bytes: |
283 | | * One trail byte state that results in code points, and one that only |
284 | | * has "unassigned" and "illegal" terminal states. |
285 | | * |
286 | | * Note: partly by accident, this data structure supports simple stateful |
287 | | * encodings without any additional logic. |
288 | | * Currently, only simple Shift-In/Shift-Out schemes are handled with |
289 | | * appropriate state tables (especially EBCDIC_STATEFUL!). |
290 | | * |
291 | | * MBCS version 2 added: |
292 | | * unassigned and illegal action codes have U+fffe and U+ffff |
293 | | * instead of unused bits; this is useful for _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP() |
294 | | * |
295 | | * Converting from Unicode to codepage bytes --------------------------------*** |
296 | | * |
297 | | * The conversion data structure for fromUnicode is designed for the known |
298 | | * structure of Unicode. It maps from 21-bit code points (0..0x10ffff) to |
299 | | * a sequence of 1..4 bytes, in addition to a flag that indicates if there is |
300 | | * a roundtrip mapping. |
301 | | * |
302 | | * The lookup is done with a 3-stage trie, using 11/6/4 bits for stage 1/2/3 |
303 | | * like in the character properties table. |
304 | | * The beginning of the trie is at offsetFromUTable, the beginning of stage 3 |
305 | | * with the resulting bytes is at offsetFromUBytes. |
306 | | * |
307 | | * Beginning with version 4, single-byte codepages have a significantly different |
308 | | * trie compared to other codepages. |
309 | | * In all cases, the entry in stage 1 is directly the index of the block of |
310 | | * 64 entries in stage 2. |
311 | | * |
312 | | * Single-byte lookup: |
313 | | * |
314 | | * Stage 2 only contains 16-bit indexes directly to the 16-blocks in stage 3. |
315 | | * Stage 3 contains one 16-bit word per result: |
316 | | * Bits 15..8 indicate the kind of result: |
317 | | * f roundtrip result |
318 | | * c fallback result from private-use code point |
319 | | * 8 fallback result from other code points |
320 | | * 0 unassigned |
321 | | * Bits 7..0 contain the codepage byte. A zero byte is always possible. |
322 | | * |
323 | | * In version 4.3, the runtime code can build an sbcsIndex for a utf8Friendly |
324 | | * file. For 2-byte UTF-8 byte sequences and some 3-byte sequences the lookup |
325 | | * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. |
326 | | * ASCII code points can be looked up with a linear array access into stage 3. |
327 | | * See maxFastUChar and other details in ucnvmbcs.h. |
328 | | * |
329 | | * Multi-byte lookup: |
330 | | * |
331 | | * Stage 2 contains a 32-bit word for each 16-block in stage 3: |
332 | | * Bits 31..16 contain flags for which stage 3 entries contain roundtrip results |
333 | | * test: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) |
334 | | * If this test is false, then a non-zero result will be interpreted as |
335 | | * a fallback mapping. |
336 | | * Bits 15..0 contain the index to stage 3, which must be multiplied by 16*(bytes per char) |
337 | | * |
338 | | * Stage 3 contains 2, 3, or 4 bytes per result. |
339 | | * 2 or 4 bytes are stored as uint16_t/uint32_t in platform endianness, |
340 | | * while 3 bytes are stored as bytes in big-endian order. |
341 | | * Leading zero bytes are ignored, and the number of bytes is counted. |
342 | | * A zero byte mapping result is possible as a roundtrip result. |
343 | | * For some output types, the actual result is processed from this; |
344 | | * see ucnv_MBCSFromUnicodeWithOffsets(). |
345 | | * |
346 | | * Note that stage 1 always contains 0x440=1088 entries (0x440==0x110000>>10), |
347 | | * or (version 3 and up) for BMP-only codepages, it contains 64 entries. |
348 | | * |
349 | | * In version 4.3, a utf8Friendly file contains an mbcsIndex table. |
350 | | * For 2-byte UTF-8 byte sequences and most 3-byte sequences the lookup |
351 | | * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. |
352 | | * ASCII code points can be looked up with a linear array access into stage 3. |
353 | | * See maxFastUChar, mbcsIndex and other details in ucnvmbcs.h. |
354 | | * |
355 | | * In version 3, stage 2 blocks may overlap by multiples of the multiplier |
356 | | * for compaction. |
357 | | * In version 4, stage 2 blocks (and for single-byte codepages, stage 3 blocks) |
358 | | * may overlap by any number of entries. |
359 | | * |
360 | | * MBCS version 2 added: |
361 | | * the converter checks for known output types, which allows |
362 | | * adding new ones without crashing an unaware converter |
363 | | */ |
364 | | |
365 | | /** |
366 | | * Callback from ucnv_MBCSEnumToUnicode(), takes 32 mappings from |
367 | | * consecutive sequences of bytes, starting from the one encoded in value, |
368 | | * to Unicode code points. (Multiple mappings to reduce per-function call overhead.) |
369 | | * Does not currently support m:n mappings or reverse fallbacks. |
370 | | * This function will not be called for sequences of bytes with leading zeros. |
371 | | * |
372 | | * @param context an opaque pointer, as passed into ucnv_MBCSEnumToUnicode() |
373 | | * @param value contains 1..4 bytes of the first byte sequence, right-aligned |
374 | | * @param codePoints resulting Unicode code points, or negative if a byte sequence does |
375 | | * not map to anything |
376 | | * @return TRUE to continue enumeration, FALSE to stop |
377 | | */ |
378 | | typedef UBool U_CALLCONV |
379 | | UConverterEnumToUCallback(const void *context, uint32_t value, UChar32 codePoints[32]); |
380 | | |
381 | | static void U_CALLCONV |
382 | | ucnv_MBCSLoad(UConverterSharedData *sharedData, |
383 | | UConverterLoadArgs *pArgs, |
384 | | const uint8_t *raw, |
385 | | UErrorCode *pErrorCode); |
386 | | |
387 | | static void U_CALLCONV |
388 | | ucnv_MBCSUnload(UConverterSharedData *sharedData); |
389 | | |
390 | | static void U_CALLCONV |
391 | | ucnv_MBCSOpen(UConverter *cnv, |
392 | | UConverterLoadArgs *pArgs, |
393 | | UErrorCode *pErrorCode); |
394 | | |
395 | | static UChar32 U_CALLCONV |
396 | | ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, |
397 | | UErrorCode *pErrorCode); |
398 | | |
399 | | static void U_CALLCONV |
400 | | ucnv_MBCSGetStarters(const UConverter* cnv, |
401 | | UBool starters[256], |
402 | | UErrorCode *pErrorCode); |
403 | | |
404 | | U_CDECL_BEGIN |
405 | | static const char* U_CALLCONV |
406 | | ucnv_MBCSGetName(const UConverter *cnv); |
407 | | U_CDECL_END |
408 | | |
409 | | static void U_CALLCONV |
410 | | ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs, |
411 | | int32_t offsetIndex, |
412 | | UErrorCode *pErrorCode); |
413 | | |
414 | | static UChar32 U_CALLCONV |
415 | | ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, |
416 | | UErrorCode *pErrorCode); |
417 | | |
418 | | static void U_CALLCONV |
419 | | ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, |
420 | | UConverterToUnicodeArgs *pToUArgs, |
421 | | UErrorCode *pErrorCode); |
422 | | |
423 | | static void U_CALLCONV |
424 | | ucnv_MBCSGetUnicodeSet(const UConverter *cnv, |
425 | | const USetAdder *sa, |
426 | | UConverterUnicodeSet which, |
427 | | UErrorCode *pErrorCode); |
428 | | |
429 | | static void U_CALLCONV |
430 | | ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, |
431 | | UConverterToUnicodeArgs *pToUArgs, |
432 | | UErrorCode *pErrorCode); |
433 | | |
434 | | static const UConverterImpl _SBCSUTF8Impl={ |
435 | | UCNV_MBCS, |
436 | | |
437 | | ucnv_MBCSLoad, |
438 | | ucnv_MBCSUnload, |
439 | | |
440 | | ucnv_MBCSOpen, |
441 | | NULL, |
442 | | NULL, |
443 | | |
444 | | ucnv_MBCSToUnicodeWithOffsets, |
445 | | ucnv_MBCSToUnicodeWithOffsets, |
446 | | ucnv_MBCSFromUnicodeWithOffsets, |
447 | | ucnv_MBCSFromUnicodeWithOffsets, |
448 | | ucnv_MBCSGetNextUChar, |
449 | | |
450 | | ucnv_MBCSGetStarters, |
451 | | ucnv_MBCSGetName, |
452 | | ucnv_MBCSWriteSub, |
453 | | NULL, |
454 | | ucnv_MBCSGetUnicodeSet, |
455 | | |
456 | | NULL, |
457 | | ucnv_SBCSFromUTF8 |
458 | | }; |
459 | | |
460 | | static const UConverterImpl _DBCSUTF8Impl={ |
461 | | UCNV_MBCS, |
462 | | |
463 | | ucnv_MBCSLoad, |
464 | | ucnv_MBCSUnload, |
465 | | |
466 | | ucnv_MBCSOpen, |
467 | | NULL, |
468 | | NULL, |
469 | | |
470 | | ucnv_MBCSToUnicodeWithOffsets, |
471 | | ucnv_MBCSToUnicodeWithOffsets, |
472 | | ucnv_MBCSFromUnicodeWithOffsets, |
473 | | ucnv_MBCSFromUnicodeWithOffsets, |
474 | | ucnv_MBCSGetNextUChar, |
475 | | |
476 | | ucnv_MBCSGetStarters, |
477 | | ucnv_MBCSGetName, |
478 | | ucnv_MBCSWriteSub, |
479 | | NULL, |
480 | | ucnv_MBCSGetUnicodeSet, |
481 | | |
482 | | NULL, |
483 | | ucnv_DBCSFromUTF8 |
484 | | }; |
485 | | |
486 | | static const UConverterImpl _MBCSImpl={ |
487 | | UCNV_MBCS, |
488 | | |
489 | | ucnv_MBCSLoad, |
490 | | ucnv_MBCSUnload, |
491 | | |
492 | | ucnv_MBCSOpen, |
493 | | NULL, |
494 | | NULL, |
495 | | |
496 | | ucnv_MBCSToUnicodeWithOffsets, |
497 | | ucnv_MBCSToUnicodeWithOffsets, |
498 | | ucnv_MBCSFromUnicodeWithOffsets, |
499 | | ucnv_MBCSFromUnicodeWithOffsets, |
500 | | ucnv_MBCSGetNextUChar, |
501 | | |
502 | | ucnv_MBCSGetStarters, |
503 | | ucnv_MBCSGetName, |
504 | | ucnv_MBCSWriteSub, |
505 | | NULL, |
506 | | ucnv_MBCSGetUnicodeSet, |
507 | | NULL, |
508 | | NULL |
509 | | }; |
510 | | |
511 | | /* Static data is in tools/makeconv/ucnvstat.c for data-based |
512 | | * converters. Be sure to update it as well. |
513 | | */ |
514 | | |
515 | | const UConverterSharedData _MBCSData={ |
516 | | sizeof(UConverterSharedData), 1, |
517 | | NULL, NULL, FALSE, TRUE, &_MBCSImpl, |
518 | | 0, UCNV_MBCS_TABLE_INITIALIZER |
519 | | }; |
520 | | |
521 | | |
522 | | /* GB 18030 data ------------------------------------------------------------ */ |
523 | | |
524 | | /* helper macros for linear values for GB 18030 four-byte sequences */ |
525 | 0 | #define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d)) |
526 | | |
527 | 0 | #define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30) |
528 | | |
529 | | #define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff) |
530 | | |
531 | | /* |
532 | | * Some ranges of GB 18030 where both the Unicode code points and the |
533 | | * GB four-byte sequences are contiguous and are handled algorithmically by |
534 | | * the special callback functions below. |
535 | | * The values are start & end of Unicode & GB codes. |
536 | | * |
537 | | * Note that single surrogates are not mapped by GB 18030 |
538 | | * as of the re-released mapping tables from 2000-nov-30. |
539 | | */ |
540 | | static const uint32_t |
541 | | gb18030Ranges[14][4]={ |
542 | | {0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35)}, |
543 | | {0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738)}, |
544 | | {0x0452, 0x1E3E, LINEAR(0x8130D330), LINEAR(0x8135F436)}, |
545 | | {0x1E40, 0x200F, LINEAR(0x8135F438), LINEAR(0x8136A531)}, |
546 | | {0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534)}, |
547 | | {0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38)}, |
548 | | {0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537)}, |
549 | | {0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32)}, |
550 | | {0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237)}, |
551 | | {0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733)}, |
552 | | {0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837)}, |
553 | | {0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638)}, |
554 | | {0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931)}, |
555 | | {0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439)} |
556 | | }; |
557 | | |
558 | | /* bit flag for UConverter.options indicating GB 18030 special handling */ |
559 | 0 | #define _MBCS_OPTION_GB18030 0x8000 |
560 | | |
561 | | /* bit flag for UConverter.options indicating KEIS,JEF,JIF special handling */ |
562 | 0 | #define _MBCS_OPTION_KEIS 0x01000 |
563 | 0 | #define _MBCS_OPTION_JEF 0x02000 |
564 | 0 | #define _MBCS_OPTION_JIPS 0x04000 |
565 | | |
566 | 0 | #define KEIS_SO_CHAR_1 0x0A |
567 | 0 | #define KEIS_SO_CHAR_2 0x42 |
568 | 0 | #define KEIS_SI_CHAR_1 0x0A |
569 | 0 | #define KEIS_SI_CHAR_2 0x41 |
570 | | |
571 | 0 | #define JEF_SO_CHAR 0x28 |
572 | 0 | #define JEF_SI_CHAR 0x29 |
573 | | |
574 | 0 | #define JIPS_SO_CHAR_1 0x1A |
575 | 0 | #define JIPS_SO_CHAR_2 0x70 |
576 | 0 | #define JIPS_SI_CHAR_1 0x1A |
577 | 0 | #define JIPS_SI_CHAR_2 0x71 |
578 | | |
579 | | enum SISO_Option { |
580 | | SI, |
581 | | SO |
582 | | }; |
583 | | typedef enum SISO_Option SISO_Option; |
584 | | |
585 | 0 | static int32_t getSISOBytes(SISO_Option option, uint32_t cnvOption, uint8_t *value) { |
586 | 0 | int32_t SISOLength = 0; |
587 | |
|
588 | 0 | switch (option) { |
589 | 0 | case SI: |
590 | 0 | if ((cnvOption&_MBCS_OPTION_KEIS)!=0) { |
591 | 0 | value[0] = KEIS_SI_CHAR_1; |
592 | 0 | value[1] = KEIS_SI_CHAR_2; |
593 | 0 | SISOLength = 2; |
594 | 0 | } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) { |
595 | 0 | value[0] = JEF_SI_CHAR; |
596 | 0 | SISOLength = 1; |
597 | 0 | } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) { |
598 | 0 | value[0] = JIPS_SI_CHAR_1; |
599 | 0 | value[1] = JIPS_SI_CHAR_2; |
600 | 0 | SISOLength = 2; |
601 | 0 | } else { |
602 | 0 | value[0] = UCNV_SI; |
603 | 0 | SISOLength = 1; |
604 | 0 | } |
605 | 0 | break; |
606 | 0 | case SO: |
607 | 0 | if ((cnvOption&_MBCS_OPTION_KEIS)!=0) { |
608 | 0 | value[0] = KEIS_SO_CHAR_1; |
609 | 0 | value[1] = KEIS_SO_CHAR_2; |
610 | 0 | SISOLength = 2; |
611 | 0 | } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) { |
612 | 0 | value[0] = JEF_SO_CHAR; |
613 | 0 | SISOLength = 1; |
614 | 0 | } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) { |
615 | 0 | value[0] = JIPS_SO_CHAR_1; |
616 | 0 | value[1] = JIPS_SO_CHAR_2; |
617 | 0 | SISOLength = 2; |
618 | 0 | } else { |
619 | 0 | value[0] = UCNV_SO; |
620 | 0 | SISOLength = 1; |
621 | 0 | } |
622 | 0 | break; |
623 | 0 | default: |
624 | | /* Should never happen. */ |
625 | 0 | break; |
626 | 0 | } |
627 | | |
628 | 0 | return SISOLength; |
629 | 0 | } |
630 | | |
631 | | /* Miscellaneous ------------------------------------------------------------ */ |
632 | | |
633 | | /* similar to ucnv_MBCSGetNextUChar() but recursive */ |
634 | | static UBool |
635 | | enumToU(UConverterMBCSTable *mbcsTable, int8_t stateProps[], |
636 | | int32_t state, uint32_t offset, |
637 | | uint32_t value, |
638 | | UConverterEnumToUCallback *callback, const void *context, |
639 | 0 | UErrorCode *pErrorCode) { |
640 | 0 | UChar32 codePoints[32]; |
641 | 0 | const int32_t *row; |
642 | 0 | const uint16_t *unicodeCodeUnits; |
643 | 0 | UChar32 anyCodePoints; |
644 | 0 | int32_t b, limit; |
645 | |
|
646 | 0 | row=mbcsTable->stateTable[state]; |
647 | 0 | unicodeCodeUnits=mbcsTable->unicodeCodeUnits; |
648 | |
|
649 | 0 | value<<=8; |
650 | 0 | anyCodePoints=-1; /* becomes non-negative if there is a mapping */ |
651 | |
|
652 | 0 | b=(stateProps[state]&0x38)<<2; |
653 | 0 | if(b==0 && stateProps[state]>=0x40) { |
654 | | /* skip byte sequences with leading zeros because they are not stored in the fromUnicode table */ |
655 | 0 | codePoints[0]=U_SENTINEL; |
656 | 0 | b=1; |
657 | 0 | } |
658 | 0 | limit=((stateProps[state]&7)+1)<<5; |
659 | 0 | while(b<limit) { |
660 | 0 | int32_t entry=row[b]; |
661 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
662 | 0 | int32_t nextState=MBCS_ENTRY_TRANSITION_STATE(entry); |
663 | 0 | if(stateProps[nextState]>=0) { |
664 | | /* recurse to a state with non-ignorable actions */ |
665 | 0 | if(!enumToU( |
666 | 0 | mbcsTable, stateProps, nextState, |
667 | 0 | offset+MBCS_ENTRY_TRANSITION_OFFSET(entry), |
668 | 0 | value|(uint32_t)b, |
669 | 0 | callback, context, |
670 | 0 | pErrorCode)) { |
671 | 0 | return FALSE; |
672 | 0 | } |
673 | 0 | } |
674 | 0 | codePoints[b&0x1f]=U_SENTINEL; |
675 | 0 | } else { |
676 | 0 | UChar32 c; |
677 | 0 | int32_t action; |
678 | | |
679 | | /* |
680 | | * An if-else-if chain provides more reliable performance for |
681 | | * the most common cases compared to a switch. |
682 | | */ |
683 | 0 | action=MBCS_ENTRY_FINAL_ACTION(entry); |
684 | 0 | if(action==MBCS_STATE_VALID_DIRECT_16) { |
685 | | /* output BMP code point */ |
686 | 0 | c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
687 | 0 | } else if(action==MBCS_STATE_VALID_16) { |
688 | 0 | int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); |
689 | 0 | c=unicodeCodeUnits[finalOffset]; |
690 | 0 | if(c<0xfffe) { |
691 | | /* output BMP code point */ |
692 | 0 | } else { |
693 | 0 | c=U_SENTINEL; |
694 | 0 | } |
695 | 0 | } else if(action==MBCS_STATE_VALID_16_PAIR) { |
696 | 0 | int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); |
697 | 0 | c=unicodeCodeUnits[finalOffset++]; |
698 | 0 | if(c<0xd800) { |
699 | | /* output BMP code point below 0xd800 */ |
700 | 0 | } else if(c<=0xdbff) { |
701 | | /* output roundtrip or fallback supplementary code point */ |
702 | 0 | c=((c&0x3ff)<<10)+unicodeCodeUnits[finalOffset]+(0x10000-0xdc00); |
703 | 0 | } else if(c==0xe000) { |
704 | | /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ |
705 | 0 | c=unicodeCodeUnits[finalOffset]; |
706 | 0 | } else { |
707 | 0 | c=U_SENTINEL; |
708 | 0 | } |
709 | 0 | } else if(action==MBCS_STATE_VALID_DIRECT_20) { |
710 | | /* output supplementary code point */ |
711 | 0 | c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); |
712 | 0 | } else { |
713 | 0 | c=U_SENTINEL; |
714 | 0 | } |
715 | |
|
716 | 0 | codePoints[b&0x1f]=c; |
717 | 0 | anyCodePoints&=c; |
718 | 0 | } |
719 | 0 | if(((++b)&0x1f)==0) { |
720 | 0 | if(anyCodePoints>=0) { |
721 | 0 | if(!callback(context, value|(uint32_t)(b-0x20), codePoints)) { |
722 | 0 | return FALSE; |
723 | 0 | } |
724 | 0 | anyCodePoints=-1; |
725 | 0 | } |
726 | 0 | } |
727 | 0 | } |
728 | 0 | return TRUE; |
729 | 0 | } |
730 | | |
731 | | /* |
732 | | * Only called if stateProps[state]==-1. |
733 | | * A recursive call may do stateProps[state]|=0x40 if this state is the target of an |
734 | | * MBCS_STATE_CHANGE_ONLY. |
735 | | */ |
736 | | static int8_t |
737 | 0 | getStateProp(const int32_t (*stateTable)[256], int8_t stateProps[], int state) { |
738 | 0 | const int32_t *row; |
739 | 0 | int32_t min, max, entry, nextState; |
740 | |
|
741 | 0 | row=stateTable[state]; |
742 | 0 | stateProps[state]=0; |
743 | | |
744 | | /* find first non-ignorable state */ |
745 | 0 | for(min=0;; ++min) { |
746 | 0 | entry=row[min]; |
747 | 0 | nextState=MBCS_ENTRY_STATE(entry); |
748 | 0 | if(stateProps[nextState]==-1) { |
749 | 0 | getStateProp(stateTable, stateProps, nextState); |
750 | 0 | } |
751 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
752 | 0 | if(stateProps[nextState]>=0) { |
753 | 0 | break; |
754 | 0 | } |
755 | 0 | } else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) { |
756 | 0 | break; |
757 | 0 | } |
758 | 0 | if(min==0xff) { |
759 | 0 | stateProps[state]=-0x40; /* (int8_t)0xc0 */ |
760 | 0 | return stateProps[state]; |
761 | 0 | } |
762 | 0 | } |
763 | 0 | stateProps[state]|=(int8_t)((min>>5)<<3); |
764 | | |
765 | | /* find last non-ignorable state */ |
766 | 0 | for(max=0xff; min<max; --max) { |
767 | 0 | entry=row[max]; |
768 | 0 | nextState=MBCS_ENTRY_STATE(entry); |
769 | 0 | if(stateProps[nextState]==-1) { |
770 | 0 | getStateProp(stateTable, stateProps, nextState); |
771 | 0 | } |
772 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
773 | 0 | if(stateProps[nextState]>=0) { |
774 | 0 | break; |
775 | 0 | } |
776 | 0 | } else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) { |
777 | 0 | break; |
778 | 0 | } |
779 | 0 | } |
780 | 0 | stateProps[state]|=(int8_t)(max>>5); |
781 | | |
782 | | /* recurse further and collect direct-state information */ |
783 | 0 | while(min<=max) { |
784 | 0 | entry=row[min]; |
785 | 0 | nextState=MBCS_ENTRY_STATE(entry); |
786 | 0 | if(stateProps[nextState]==-1) { |
787 | 0 | getStateProp(stateTable, stateProps, nextState); |
788 | 0 | } |
789 | 0 | if(MBCS_ENTRY_IS_FINAL(entry)) { |
790 | 0 | stateProps[nextState]|=0x40; |
791 | 0 | if(MBCS_ENTRY_FINAL_ACTION(entry)<=MBCS_STATE_FALLBACK_DIRECT_20) { |
792 | 0 | stateProps[state]|=0x40; |
793 | 0 | } |
794 | 0 | } |
795 | 0 | ++min; |
796 | 0 | } |
797 | 0 | return stateProps[state]; |
798 | 0 | } |
799 | | |
800 | | /* |
801 | | * Internal function enumerating the toUnicode data of an MBCS converter. |
802 | | * Currently only used for reconstituting data for a MBCS_OPT_NO_FROM_U |
803 | | * table, but could also be used for a future ucnv_getUnicodeSet() option |
804 | | * that includes reverse fallbacks (after updating this function's implementation). |
805 | | * Currently only handles roundtrip mappings. |
806 | | * Does not currently handle extensions. |
807 | | */ |
808 | | static void |
809 | | ucnv_MBCSEnumToUnicode(UConverterMBCSTable *mbcsTable, |
810 | | UConverterEnumToUCallback *callback, const void *context, |
811 | 0 | UErrorCode *pErrorCode) { |
812 | | /* |
813 | | * Properties for each state, to speed up the enumeration. |
814 | | * Ignorable actions are unassigned/illegal/state-change-only: |
815 | | * They do not lead to mappings. |
816 | | * |
817 | | * Bits 7..6: |
818 | | * 1 direct/initial state (stateful converters have multiple) |
819 | | * 0 non-initial state with transitions or with non-ignorable result actions |
820 | | * -1 final state with only ignorable actions |
821 | | * |
822 | | * Bits 5..3: |
823 | | * The lowest byte value with non-ignorable actions is |
824 | | * value<<5 (rounded down). |
825 | | * |
826 | | * Bits 2..0: |
827 | | * The highest byte value with non-ignorable actions is |
828 | | * (value<<5)&0x1f (rounded up). |
829 | | */ |
830 | 0 | int8_t stateProps[MBCS_MAX_STATE_COUNT]; |
831 | 0 | int32_t state; |
832 | |
|
833 | 0 | uprv_memset(stateProps, -1, sizeof(stateProps)); |
834 | | |
835 | | /* recurse from state 0 and set all stateProps */ |
836 | 0 | getStateProp(mbcsTable->stateTable, stateProps, 0); |
837 | |
|
838 | 0 | for(state=0; state<mbcsTable->countStates; ++state) { |
839 | | /*if(stateProps[state]==-1) { |
840 | | printf("unused/unreachable <icu:state> %d\n", state); |
841 | | }*/ |
842 | 0 | if(stateProps[state]>=0x40) { |
843 | | /* start from each direct state */ |
844 | 0 | enumToU( |
845 | 0 | mbcsTable, stateProps, state, 0, 0, |
846 | 0 | callback, context, |
847 | 0 | pErrorCode); |
848 | 0 | } |
849 | 0 | } |
850 | 0 | } |
851 | | |
852 | | U_CFUNC void |
853 | | ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, |
854 | | const USetAdder *sa, |
855 | | UConverterUnicodeSet which, |
856 | | UConverterSetFilter filter, |
857 | 0 | UErrorCode *pErrorCode) { |
858 | 0 | const UConverterMBCSTable *mbcsTable; |
859 | 0 | const uint16_t *table; |
860 | |
|
861 | 0 | uint32_t st3; |
862 | 0 | uint16_t st1, maxStage1, st2; |
863 | |
|
864 | 0 | UChar32 c; |
865 | | |
866 | | /* enumerate the from-Unicode trie table */ |
867 | 0 | mbcsTable=&sharedData->mbcs; |
868 | 0 | table=mbcsTable->fromUnicodeTable; |
869 | 0 | if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { |
870 | 0 | maxStage1=0x440; |
871 | 0 | } else { |
872 | 0 | maxStage1=0x40; |
873 | 0 | } |
874 | |
|
875 | 0 | c=0; /* keep track of the current code point while enumerating */ |
876 | |
|
877 | 0 | if(mbcsTable->outputType==MBCS_OUTPUT_1) { |
878 | 0 | const uint16_t *stage2, *stage3, *results; |
879 | 0 | uint16_t minValue; |
880 | |
|
881 | 0 | results=(const uint16_t *)mbcsTable->fromUnicodeBytes; |
882 | | |
883 | | /* |
884 | | * Set a threshold variable for selecting which mappings to use. |
885 | | * See ucnv_MBCSSingleFromBMPWithOffsets() and |
886 | | * MBCS_SINGLE_RESULT_FROM_U() for details. |
887 | | */ |
888 | 0 | if(which==UCNV_ROUNDTRIP_SET) { |
889 | | /* use only roundtrips */ |
890 | 0 | minValue=0xf00; |
891 | 0 | } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ { |
892 | | /* use all roundtrip and fallback results */ |
893 | 0 | minValue=0x800; |
894 | 0 | } |
895 | |
|
896 | 0 | for(st1=0; st1<maxStage1; ++st1) { |
897 | 0 | st2=table[st1]; |
898 | 0 | if(st2>maxStage1) { |
899 | 0 | stage2=table+st2; |
900 | 0 | for(st2=0; st2<64; ++st2) { |
901 | 0 | if((st3=stage2[st2])!=0) { |
902 | | /* read the stage 3 block */ |
903 | 0 | stage3=results+st3; |
904 | |
|
905 | 0 | do { |
906 | 0 | if(*stage3++>=minValue) { |
907 | 0 | sa->add(sa->set, c); |
908 | 0 | } |
909 | 0 | } while((++c&0xf)!=0); |
910 | 0 | } else { |
911 | 0 | c+=16; /* empty stage 3 block */ |
912 | 0 | } |
913 | 0 | } |
914 | 0 | } else { |
915 | 0 | c+=1024; /* empty stage 2 block */ |
916 | 0 | } |
917 | 0 | } |
918 | 0 | } else { |
919 | 0 | const uint32_t *stage2; |
920 | 0 | const uint8_t *stage3, *bytes; |
921 | 0 | uint32_t st3Multiplier; |
922 | 0 | uint32_t value; |
923 | 0 | UBool useFallback; |
924 | |
|
925 | 0 | bytes=mbcsTable->fromUnicodeBytes; |
926 | |
|
927 | 0 | useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET); |
928 | |
|
929 | 0 | switch(mbcsTable->outputType) { |
930 | 0 | case MBCS_OUTPUT_3: |
931 | 0 | case MBCS_OUTPUT_4_EUC: |
932 | 0 | st3Multiplier=3; |
933 | 0 | break; |
934 | 0 | case MBCS_OUTPUT_4: |
935 | 0 | st3Multiplier=4; |
936 | 0 | break; |
937 | 0 | default: |
938 | 0 | st3Multiplier=2; |
939 | 0 | break; |
940 | 0 | } |
941 | | |
942 | 0 | for(st1=0; st1<maxStage1; ++st1) { |
943 | 0 | st2=table[st1]; |
944 | 0 | if(st2>(maxStage1>>1)) { |
945 | 0 | stage2=(const uint32_t *)table+st2; |
946 | 0 | for(st2=0; st2<64; ++st2) { |
947 | 0 | if((st3=stage2[st2])!=0) { |
948 | | /* read the stage 3 block */ |
949 | 0 | stage3=bytes+st3Multiplier*16*(uint32_t)(uint16_t)st3; |
950 | | |
951 | | /* get the roundtrip flags for the stage 3 block */ |
952 | 0 | st3>>=16; |
953 | | |
954 | | /* |
955 | | * Add code points for which the roundtrip flag is set, |
956 | | * or which map to non-zero bytes if we use fallbacks. |
957 | | * See ucnv_MBCSFromUnicodeWithOffsets() for details. |
958 | | */ |
959 | 0 | switch(filter) { |
960 | 0 | case UCNV_SET_FILTER_NONE: |
961 | 0 | do { |
962 | 0 | if(st3&1) { |
963 | 0 | sa->add(sa->set, c); |
964 | 0 | stage3+=st3Multiplier; |
965 | 0 | } else if(useFallback) { |
966 | 0 | uint8_t b=0; |
967 | 0 | switch(st3Multiplier) { |
968 | 0 | case 4: |
969 | 0 | b|=*stage3++; |
970 | 0 | U_FALLTHROUGH; |
971 | 0 | case 3: |
972 | 0 | b|=*stage3++; |
973 | 0 | U_FALLTHROUGH; |
974 | 0 | case 2: |
975 | 0 | b|=stage3[0]|stage3[1]; |
976 | 0 | stage3+=2; |
977 | 0 | U_FALLTHROUGH; |
978 | 0 | default: |
979 | 0 | break; |
980 | 0 | } |
981 | 0 | if(b!=0) { |
982 | 0 | sa->add(sa->set, c); |
983 | 0 | } |
984 | 0 | } |
985 | 0 | st3>>=1; |
986 | 0 | } while((++c&0xf)!=0); |
987 | 0 | break; |
988 | 0 | case UCNV_SET_FILTER_DBCS_ONLY: |
989 | | /* Ignore single-byte results (<0x100). */ |
990 | 0 | do { |
991 | 0 | if(((st3&1)!=0 || useFallback) && *((const uint16_t *)stage3)>=0x100) { |
992 | 0 | sa->add(sa->set, c); |
993 | 0 | } |
994 | 0 | st3>>=1; |
995 | 0 | stage3+=2; /* +=st3Multiplier */ |
996 | 0 | } while((++c&0xf)!=0); |
997 | 0 | break; |
998 | 0 | case UCNV_SET_FILTER_2022_CN: |
999 | | /* Only add code points that map to CNS 11643 planes 1 & 2 for non-EXT ISO-2022-CN. */ |
1000 | 0 | do { |
1001 | 0 | if(((st3&1)!=0 || useFallback) && ((value=*stage3)==0x81 || value==0x82)) { |
1002 | 0 | sa->add(sa->set, c); |
1003 | 0 | } |
1004 | 0 | st3>>=1; |
1005 | 0 | stage3+=3; /* +=st3Multiplier */ |
1006 | 0 | } while((++c&0xf)!=0); |
1007 | 0 | break; |
1008 | 0 | case UCNV_SET_FILTER_SJIS: |
1009 | | /* Only add code points that map to Shift-JIS codes corresponding to JIS X 0208. */ |
1010 | 0 | do { |
1011 | 0 | if(((st3&1)!=0 || useFallback) && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) { |
1012 | 0 | sa->add(sa->set, c); |
1013 | 0 | } |
1014 | 0 | st3>>=1; |
1015 | 0 | stage3+=2; /* +=st3Multiplier */ |
1016 | 0 | } while((++c&0xf)!=0); |
1017 | 0 | break; |
1018 | 0 | case UCNV_SET_FILTER_GR94DBCS: |
1019 | | /* Only add code points that map to ISO 2022 GR 94 DBCS codes (each byte A1..FE). */ |
1020 | 0 | do { |
1021 | 0 | if( ((st3&1)!=0 || useFallback) && |
1022 | 0 | (uint16_t)((value=*((const uint16_t *)stage3)) - 0xa1a1)<=(0xfefe - 0xa1a1) && |
1023 | 0 | (uint8_t)(value-0xa1)<=(0xfe - 0xa1) |
1024 | 0 | ) { |
1025 | 0 | sa->add(sa->set, c); |
1026 | 0 | } |
1027 | 0 | st3>>=1; |
1028 | 0 | stage3+=2; /* +=st3Multiplier */ |
1029 | 0 | } while((++c&0xf)!=0); |
1030 | 0 | break; |
1031 | 0 | case UCNV_SET_FILTER_HZ: |
1032 | | /* Only add code points that are suitable for HZ DBCS (lead byte A1..FD). */ |
1033 | 0 | do { |
1034 | 0 | if( ((st3&1)!=0 || useFallback) && |
1035 | 0 | (uint16_t)((value=*((const uint16_t *)stage3))-0xa1a1)<=(0xfdfe - 0xa1a1) && |
1036 | 0 | (uint8_t)(value-0xa1)<=(0xfe - 0xa1) |
1037 | 0 | ) { |
1038 | 0 | sa->add(sa->set, c); |
1039 | 0 | } |
1040 | 0 | st3>>=1; |
1041 | 0 | stage3+=2; /* +=st3Multiplier */ |
1042 | 0 | } while((++c&0xf)!=0); |
1043 | 0 | break; |
1044 | 0 | default: |
1045 | 0 | *pErrorCode=U_INTERNAL_PROGRAM_ERROR; |
1046 | 0 | return; |
1047 | 0 | } |
1048 | 0 | } else { |
1049 | 0 | c+=16; /* empty stage 3 block */ |
1050 | 0 | } |
1051 | 0 | } |
1052 | 0 | } else { |
1053 | 0 | c+=1024; /* empty stage 2 block */ |
1054 | 0 | } |
1055 | 0 | } |
1056 | 0 | } |
1057 | | |
1058 | 0 | ucnv_extGetUnicodeSet(sharedData, sa, which, filter, pErrorCode); |
1059 | 0 | } |
1060 | | |
1061 | | U_CFUNC void |
1062 | | ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, |
1063 | | const USetAdder *sa, |
1064 | | UConverterUnicodeSet which, |
1065 | 0 | UErrorCode *pErrorCode) { |
1066 | 0 | ucnv_MBCSGetFilteredUnicodeSetForUnicode( |
1067 | 0 | sharedData, sa, which, |
1068 | 0 | sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? |
1069 | 0 | UCNV_SET_FILTER_DBCS_ONLY : |
1070 | 0 | UCNV_SET_FILTER_NONE, |
1071 | 0 | pErrorCode); |
1072 | 0 | } |
1073 | | |
1074 | | static void U_CALLCONV |
1075 | | ucnv_MBCSGetUnicodeSet(const UConverter *cnv, |
1076 | | const USetAdder *sa, |
1077 | | UConverterUnicodeSet which, |
1078 | 0 | UErrorCode *pErrorCode) { |
1079 | 0 | if(cnv->options&_MBCS_OPTION_GB18030) { |
1080 | 0 | sa->addRange(sa->set, 0, 0xd7ff); |
1081 | 0 | sa->addRange(sa->set, 0xe000, 0x10ffff); |
1082 | 0 | } else { |
1083 | 0 | ucnv_MBCSGetUnicodeSetForUnicode(cnv->sharedData, sa, which, pErrorCode); |
1084 | 0 | } |
1085 | 0 | } |
1086 | | |
1087 | | /* conversion extensions for input not in the main table -------------------- */ |
1088 | | |
1089 | | /* |
1090 | | * Hardcoded extension handling for GB 18030. |
1091 | | * Definition of LINEAR macros and gb18030Ranges see near the beginning of the file. |
1092 | | * |
1093 | | * In the future, conversion extensions may handle m:n mappings and delta tables, |
1094 | | * see https://htmlpreview.github.io/?https://github.com/unicode-org/icu-docs/blob/main/design/conversion/conversion_extensions.html |
1095 | | * |
1096 | | * If an input character cannot be mapped, then these functions set an error |
1097 | | * code. The framework will then call the callback function. |
1098 | | */ |
1099 | | |
1100 | | /* |
1101 | | * @return if(U_FAILURE) return the code point for cnv->fromUChar32 |
1102 | | * else return 0 after output has been written to the target |
1103 | | */ |
1104 | | static UChar32 |
1105 | | _extFromU(UConverter *cnv, const UConverterSharedData *sharedData, |
1106 | | UChar32 cp, |
1107 | | const UChar **source, const UChar *sourceLimit, |
1108 | | uint8_t **target, const uint8_t *targetLimit, |
1109 | | int32_t **offsets, int32_t sourceIndex, |
1110 | | UBool flush, |
1111 | 0 | UErrorCode *pErrorCode) { |
1112 | 0 | const int32_t *cx; |
1113 | |
|
1114 | 0 | cnv->useSubChar1=FALSE; |
1115 | |
|
1116 | 0 | if( (cx=sharedData->mbcs.extIndexes)!=NULL && |
1117 | 0 | ucnv_extInitialMatchFromU( |
1118 | 0 | cnv, cx, |
1119 | 0 | cp, source, sourceLimit, |
1120 | 0 | (char **)target, (char *)targetLimit, |
1121 | 0 | offsets, sourceIndex, |
1122 | 0 | flush, |
1123 | 0 | pErrorCode) |
1124 | 0 | ) { |
1125 | 0 | return 0; /* an extension mapping handled the input */ |
1126 | 0 | } |
1127 | | |
1128 | | /* GB 18030 */ |
1129 | 0 | if((cnv->options&_MBCS_OPTION_GB18030)!=0) { |
1130 | 0 | const uint32_t *range; |
1131 | 0 | int32_t i; |
1132 | |
|
1133 | 0 | range=gb18030Ranges[0]; |
1134 | 0 | for(i=0; i<UPRV_LENGTHOF(gb18030Ranges); range+=4, ++i) { |
1135 | 0 | if(range[0]<=(uint32_t)cp && (uint32_t)cp<=range[1]) { |
1136 | | /* found the Unicode code point, output the four-byte sequence for it */ |
1137 | 0 | uint32_t linear; |
1138 | 0 | char bytes[4]; |
1139 | | |
1140 | | /* get the linear value of the first GB 18030 code in this range */ |
1141 | 0 | linear=range[2]-LINEAR_18030_BASE; |
1142 | | |
1143 | | /* add the offset from the beginning of the range */ |
1144 | 0 | linear+=((uint32_t)cp-range[0]); |
1145 | | |
1146 | | /* turn this into a four-byte sequence */ |
1147 | 0 | bytes[3]=(char)(0x30+linear%10); linear/=10; |
1148 | 0 | bytes[2]=(char)(0x81+linear%126); linear/=126; |
1149 | 0 | bytes[1]=(char)(0x30+linear%10); linear/=10; |
1150 | 0 | bytes[0]=(char)(0x81+linear); |
1151 | | |
1152 | | /* output this sequence */ |
1153 | 0 | ucnv_fromUWriteBytes(cnv, |
1154 | 0 | bytes, 4, (char **)target, (char *)targetLimit, |
1155 | 0 | offsets, sourceIndex, pErrorCode); |
1156 | 0 | return 0; |
1157 | 0 | } |
1158 | 0 | } |
1159 | 0 | } |
1160 | | |
1161 | | /* no mapping */ |
1162 | 0 | *pErrorCode=U_INVALID_CHAR_FOUND; |
1163 | 0 | return cp; |
1164 | 0 | } |
1165 | | |
1166 | | /* |
1167 | | * Input sequence: cnv->toUBytes[0..length[ |
1168 | | * @return if(U_FAILURE) return the length (toULength, byteIndex) for the input |
1169 | | * else return 0 after output has been written to the target |
1170 | | */ |
1171 | | static int8_t |
1172 | | _extToU(UConverter *cnv, const UConverterSharedData *sharedData, |
1173 | | int8_t length, |
1174 | | const uint8_t **source, const uint8_t *sourceLimit, |
1175 | | UChar **target, const UChar *targetLimit, |
1176 | | int32_t **offsets, int32_t sourceIndex, |
1177 | | UBool flush, |
1178 | 0 | UErrorCode *pErrorCode) { |
1179 | 0 | const int32_t *cx; |
1180 | |
|
1181 | 0 | if( (cx=sharedData->mbcs.extIndexes)!=NULL && |
1182 | 0 | ucnv_extInitialMatchToU( |
1183 | 0 | cnv, cx, |
1184 | 0 | length, (const char **)source, (const char *)sourceLimit, |
1185 | 0 | target, targetLimit, |
1186 | 0 | offsets, sourceIndex, |
1187 | 0 | flush, |
1188 | 0 | pErrorCode) |
1189 | 0 | ) { |
1190 | 0 | return 0; /* an extension mapping handled the input */ |
1191 | 0 | } |
1192 | | |
1193 | | /* GB 18030 */ |
1194 | 0 | if(length==4 && (cnv->options&_MBCS_OPTION_GB18030)!=0) { |
1195 | 0 | const uint32_t *range; |
1196 | 0 | uint32_t linear; |
1197 | 0 | int32_t i; |
1198 | |
|
1199 | 0 | linear=LINEAR_18030(cnv->toUBytes[0], cnv->toUBytes[1], cnv->toUBytes[2], cnv->toUBytes[3]); |
1200 | 0 | range=gb18030Ranges[0]; |
1201 | 0 | for(i=0; i<UPRV_LENGTHOF(gb18030Ranges); range+=4, ++i) { |
1202 | 0 | if(range[2]<=linear && linear<=range[3]) { |
1203 | | /* found the sequence, output the Unicode code point for it */ |
1204 | 0 | *pErrorCode=U_ZERO_ERROR; |
1205 | | |
1206 | | /* add the linear difference between the input and start sequences to the start code point */ |
1207 | 0 | linear=range[0]+(linear-range[2]); |
1208 | | |
1209 | | /* output this code point */ |
1210 | 0 | ucnv_toUWriteCodePoint(cnv, linear, target, targetLimit, offsets, sourceIndex, pErrorCode); |
1211 | |
|
1212 | 0 | return 0; |
1213 | 0 | } |
1214 | 0 | } |
1215 | 0 | } |
1216 | | |
1217 | | /* no mapping */ |
1218 | 0 | *pErrorCode=U_INVALID_CHAR_FOUND; |
1219 | 0 | return length; |
1220 | 0 | } |
1221 | | |
1222 | | /* EBCDIC swap LF<->NL ------------------------------------------------------ */ |
1223 | | |
1224 | | /* |
1225 | | * This code modifies a standard EBCDIC<->Unicode mapping table for |
1226 | | * OS/390 (z/OS) Unix System Services (Open Edition). |
1227 | | * The difference is in the mapping of Line Feed and New Line control codes: |
1228 | | * Standard EBCDIC maps |
1229 | | * |
1230 | | * <U000A> \x25 |0 |
1231 | | * <U0085> \x15 |0 |
1232 | | * |
1233 | | * but OS/390 USS EBCDIC swaps the control codes for LF and NL, |
1234 | | * mapping |
1235 | | * |
1236 | | * <U000A> \x15 |0 |
1237 | | * <U0085> \x25 |0 |
1238 | | * |
1239 | | * This code modifies a loaded standard EBCDIC<->Unicode mapping table |
1240 | | * by copying it into allocated memory and swapping the LF and NL values. |
1241 | | * It allows to support the same EBCDIC charset in both versions without |
1242 | | * duplicating the entire installed table. |
1243 | | */ |
1244 | | |
1245 | | /* standard EBCDIC codes */ |
1246 | 0 | #define EBCDIC_LF 0x25 |
1247 | 0 | #define EBCDIC_NL 0x15 |
1248 | | |
1249 | | /* standard EBCDIC codes with roundtrip flag as stored in Unicode-to-single-byte tables */ |
1250 | 0 | #define EBCDIC_RT_LF 0xf25 |
1251 | 0 | #define EBCDIC_RT_NL 0xf15 |
1252 | | |
1253 | | /* Unicode code points */ |
1254 | | #define U_LF 0x0a |
1255 | | #define U_NL 0x85 |
1256 | | |
1257 | | static UBool |
1258 | 0 | _EBCDICSwapLFNL(UConverterSharedData *sharedData, UErrorCode *pErrorCode) { |
1259 | 0 | UConverterMBCSTable *mbcsTable; |
1260 | |
|
1261 | 0 | const uint16_t *table, *results; |
1262 | 0 | const uint8_t *bytes; |
1263 | |
|
1264 | 0 | int32_t (*newStateTable)[256]; |
1265 | 0 | uint16_t *newResults; |
1266 | 0 | uint8_t *p; |
1267 | 0 | char *name; |
1268 | |
|
1269 | 0 | uint32_t stage2Entry; |
1270 | 0 | uint32_t size, sizeofFromUBytes; |
1271 | |
|
1272 | 0 | mbcsTable=&sharedData->mbcs; |
1273 | |
|
1274 | 0 | table=mbcsTable->fromUnicodeTable; |
1275 | 0 | bytes=mbcsTable->fromUnicodeBytes; |
1276 | 0 | results=(const uint16_t *)bytes; |
1277 | | |
1278 | | /* |
1279 | | * Check that this is an EBCDIC table with SBCS portion - |
1280 | | * SBCS or EBCDIC_STATEFUL with standard EBCDIC LF and NL mappings. |
1281 | | * |
1282 | | * If not, ignore the option. Options are always ignored if they do not apply. |
1283 | | */ |
1284 | 0 | if(!( |
1285 | 0 | (mbcsTable->outputType==MBCS_OUTPUT_1 || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) && |
1286 | 0 | mbcsTable->stateTable[0][EBCDIC_LF]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF) && |
1287 | 0 | mbcsTable->stateTable[0][EBCDIC_NL]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL) |
1288 | 0 | )) { |
1289 | 0 | return FALSE; |
1290 | 0 | } |
1291 | | |
1292 | 0 | if(mbcsTable->outputType==MBCS_OUTPUT_1) { |
1293 | 0 | if(!( |
1294 | 0 | EBCDIC_RT_LF==MBCS_SINGLE_RESULT_FROM_U(table, results, U_LF) && |
1295 | 0 | EBCDIC_RT_NL==MBCS_SINGLE_RESULT_FROM_U(table, results, U_NL) |
1296 | 0 | )) { |
1297 | 0 | return FALSE; |
1298 | 0 | } |
1299 | 0 | } else /* MBCS_OUTPUT_2_SISO */ { |
1300 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); |
1301 | 0 | if(!( |
1302 | 0 | MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_LF)!=0 && |
1303 | 0 | EBCDIC_LF==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_LF) |
1304 | 0 | )) { |
1305 | 0 | return FALSE; |
1306 | 0 | } |
1307 | | |
1308 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); |
1309 | 0 | if(!( |
1310 | 0 | MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_NL)!=0 && |
1311 | 0 | EBCDIC_NL==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_NL) |
1312 | 0 | )) { |
1313 | 0 | return FALSE; |
1314 | 0 | } |
1315 | 0 | } |
1316 | | |
1317 | 0 | if(mbcsTable->fromUBytesLength>0) { |
1318 | | /* |
1319 | | * We _know_ the number of bytes in the fromUnicodeBytes array |
1320 | | * starting with header.version 4.1. |
1321 | | */ |
1322 | 0 | sizeofFromUBytes=mbcsTable->fromUBytesLength; |
1323 | 0 | } else { |
1324 | | /* |
1325 | | * Otherwise: |
1326 | | * There used to be code to enumerate the fromUnicode |
1327 | | * trie and find the highest entry, but it was removed in ICU 3.2 |
1328 | | * because it was not tested and caused a low code coverage number. |
1329 | | * See Jitterbug 3674. |
1330 | | * This affects only some .cnv file formats with a header.version |
1331 | | * below 4.1, and only when swaplfnl is requested. |
1332 | | * |
1333 | | * ucnvmbcs.c revision 1.99 is the last one with the |
1334 | | * ucnv_MBCSSizeofFromUBytes() function. |
1335 | | */ |
1336 | 0 | *pErrorCode=U_INVALID_FORMAT_ERROR; |
1337 | 0 | return FALSE; |
1338 | 0 | } |
1339 | | |
1340 | | /* |
1341 | | * The table has an appropriate format. |
1342 | | * Allocate and build |
1343 | | * - a modified to-Unicode state table |
1344 | | * - a modified from-Unicode output array |
1345 | | * - a converter name string with the swap option appended |
1346 | | */ |
1347 | 0 | size= |
1348 | 0 | mbcsTable->countStates*1024+ |
1349 | 0 | sizeofFromUBytes+ |
1350 | 0 | UCNV_MAX_CONVERTER_NAME_LENGTH+20; |
1351 | 0 | p=(uint8_t *)uprv_malloc(size); |
1352 | 0 | if(p==NULL) { |
1353 | 0 | *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
1354 | 0 | return FALSE; |
1355 | 0 | } |
1356 | | |
1357 | | /* copy and modify the to-Unicode state table */ |
1358 | 0 | newStateTable=(int32_t (*)[256])p; |
1359 | 0 | uprv_memcpy(newStateTable, mbcsTable->stateTable, mbcsTable->countStates*1024); |
1360 | |
|
1361 | 0 | newStateTable[0][EBCDIC_LF]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL); |
1362 | 0 | newStateTable[0][EBCDIC_NL]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF); |
1363 | | |
1364 | | /* copy and modify the from-Unicode result table */ |
1365 | 0 | newResults=(uint16_t *)newStateTable[mbcsTable->countStates]; |
1366 | 0 | uprv_memcpy(newResults, bytes, sizeofFromUBytes); |
1367 | | |
1368 | | /* conveniently, the table access macros work on the left side of expressions */ |
1369 | 0 | if(mbcsTable->outputType==MBCS_OUTPUT_1) { |
1370 | 0 | MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_LF)=EBCDIC_RT_NL; |
1371 | 0 | MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_NL)=EBCDIC_RT_LF; |
1372 | 0 | } else /* MBCS_OUTPUT_2_SISO */ { |
1373 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); |
1374 | 0 | MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_LF)=EBCDIC_NL; |
1375 | |
|
1376 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); |
1377 | 0 | MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_NL)=EBCDIC_LF; |
1378 | 0 | } |
1379 | | |
1380 | | /* set the canonical converter name */ |
1381 | 0 | name=(char *)newResults+sizeofFromUBytes; |
1382 | 0 | uprv_strcpy(name, sharedData->staticData->name); |
1383 | 0 | uprv_strcat(name, UCNV_SWAP_LFNL_OPTION_STRING); |
1384 | | |
1385 | | /* set the pointers */ |
1386 | 0 | icu::umtx_lock(NULL); |
1387 | 0 | if(mbcsTable->swapLFNLStateTable==NULL) { |
1388 | 0 | mbcsTable->swapLFNLStateTable=newStateTable; |
1389 | 0 | mbcsTable->swapLFNLFromUnicodeBytes=(uint8_t *)newResults; |
1390 | 0 | mbcsTable->swapLFNLName=name; |
1391 | |
|
1392 | 0 | newStateTable=NULL; |
1393 | 0 | } |
1394 | 0 | icu::umtx_unlock(NULL); |
1395 | | |
1396 | | /* release the allocated memory if another thread beat us to it */ |
1397 | 0 | if(newStateTable!=NULL) { |
1398 | 0 | uprv_free(newStateTable); |
1399 | 0 | } |
1400 | 0 | return TRUE; |
1401 | 0 | } |
1402 | | |
1403 | | /* reconstitute omitted fromUnicode data ------------------------------------ */ |
1404 | | |
1405 | | /* for details, compare with genmbcs.c MBCSAddFromUnicode() and transformEUC() */ |
1406 | | static UBool U_CALLCONV |
1407 | 0 | writeStage3Roundtrip(const void *context, uint32_t value, UChar32 codePoints[32]) { |
1408 | 0 | UConverterMBCSTable *mbcsTable=(UConverterMBCSTable *)context; |
1409 | 0 | const uint16_t *table; |
1410 | 0 | uint32_t *stage2; |
1411 | 0 | uint8_t *bytes, *p; |
1412 | 0 | UChar32 c; |
1413 | 0 | int32_t i, st3; |
1414 | |
|
1415 | 0 | table=mbcsTable->fromUnicodeTable; |
1416 | 0 | bytes=(uint8_t *)mbcsTable->fromUnicodeBytes; |
1417 | | |
1418 | | /* for EUC outputTypes, modify the value like genmbcs.c's transformEUC() */ |
1419 | 0 | switch(mbcsTable->outputType) { |
1420 | 0 | case MBCS_OUTPUT_3_EUC: |
1421 | 0 | if(value<=0xffff) { |
1422 | | /* short sequences are stored directly */ |
1423 | | /* code set 0 or 1 */ |
1424 | 0 | } else if(value<=0x8effff) { |
1425 | | /* code set 2 */ |
1426 | 0 | value&=0x7fff; |
1427 | 0 | } else /* first byte is 0x8f */ { |
1428 | | /* code set 3 */ |
1429 | 0 | value&=0xff7f; |
1430 | 0 | } |
1431 | 0 | break; |
1432 | 0 | case MBCS_OUTPUT_4_EUC: |
1433 | 0 | if(value<=0xffffff) { |
1434 | | /* short sequences are stored directly */ |
1435 | | /* code set 0 or 1 */ |
1436 | 0 | } else if(value<=0x8effffff) { |
1437 | | /* code set 2 */ |
1438 | 0 | value&=0x7fffff; |
1439 | 0 | } else /* first byte is 0x8f */ { |
1440 | | /* code set 3 */ |
1441 | 0 | value&=0xff7fff; |
1442 | 0 | } |
1443 | 0 | break; |
1444 | 0 | default: |
1445 | 0 | break; |
1446 | 0 | } |
1447 | | |
1448 | 0 | for(i=0; i<=0x1f; ++value, ++i) { |
1449 | 0 | c=codePoints[i]; |
1450 | 0 | if(c<0) { |
1451 | 0 | continue; |
1452 | 0 | } |
1453 | | |
1454 | | /* locate the stage 2 & 3 data */ |
1455 | 0 | stage2=((uint32_t *)table)+table[c>>10]+((c>>4)&0x3f); |
1456 | 0 | p=bytes; |
1457 | 0 | st3=(int32_t)(uint16_t)*stage2*16+(c&0xf); |
1458 | | |
1459 | | /* write the codepage bytes into stage 3 */ |
1460 | 0 | switch(mbcsTable->outputType) { |
1461 | 0 | case MBCS_OUTPUT_3: |
1462 | 0 | case MBCS_OUTPUT_4_EUC: |
1463 | 0 | p+=st3*3; |
1464 | 0 | p[0]=(uint8_t)(value>>16); |
1465 | 0 | p[1]=(uint8_t)(value>>8); |
1466 | 0 | p[2]=(uint8_t)value; |
1467 | 0 | break; |
1468 | 0 | case MBCS_OUTPUT_4: |
1469 | 0 | ((uint32_t *)p)[st3]=value; |
1470 | 0 | break; |
1471 | 0 | default: |
1472 | | /* 2 bytes per character */ |
1473 | 0 | ((uint16_t *)p)[st3]=(uint16_t)value; |
1474 | 0 | break; |
1475 | 0 | } |
1476 | | |
1477 | | /* set the roundtrip flag */ |
1478 | 0 | *stage2|=(1UL<<(16+(c&0xf))); |
1479 | 0 | } |
1480 | 0 | return TRUE; |
1481 | 0 | } |
1482 | | |
1483 | | static void |
1484 | | reconstituteData(UConverterMBCSTable *mbcsTable, |
1485 | | uint32_t stage1Length, uint32_t stage2Length, |
1486 | | uint32_t fullStage2Length, /* lengths are numbers of units, not bytes */ |
1487 | 0 | UErrorCode *pErrorCode) { |
1488 | 0 | uint16_t *stage1; |
1489 | 0 | uint32_t *stage2; |
1490 | 0 | uint32_t dataLength=stage1Length*2+fullStage2Length*4+mbcsTable->fromUBytesLength; |
1491 | 0 | mbcsTable->reconstitutedData=(uint8_t *)uprv_malloc(dataLength); |
1492 | 0 | if(mbcsTable->reconstitutedData==NULL) { |
1493 | 0 | *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
1494 | 0 | return; |
1495 | 0 | } |
1496 | 0 | uprv_memset(mbcsTable->reconstitutedData, 0, dataLength); |
1497 | | |
1498 | | /* copy existing data and reroute the pointers */ |
1499 | 0 | stage1=(uint16_t *)mbcsTable->reconstitutedData; |
1500 | 0 | uprv_memcpy(stage1, mbcsTable->fromUnicodeTable, stage1Length*2); |
1501 | |
|
1502 | 0 | stage2=(uint32_t *)(stage1+stage1Length); |
1503 | 0 | uprv_memcpy(stage2+(fullStage2Length-stage2Length), |
1504 | 0 | mbcsTable->fromUnicodeTable+stage1Length, |
1505 | 0 | stage2Length*4); |
1506 | |
|
1507 | 0 | mbcsTable->fromUnicodeTable=stage1; |
1508 | 0 | mbcsTable->fromUnicodeBytes=(uint8_t *)(stage2+fullStage2Length); |
1509 | | |
1510 | | /* indexes into stage 2 count from the bottom of the fromUnicodeTable */ |
1511 | 0 | stage2=(uint32_t *)stage1; |
1512 | | |
1513 | | /* reconstitute the initial part of stage 2 from the mbcsIndex */ |
1514 | 0 | { |
1515 | 0 | int32_t stageUTF8Length=((int32_t)mbcsTable->maxFastUChar+1)>>6; |
1516 | 0 | int32_t stageUTF8Index=0; |
1517 | 0 | int32_t st1, st2, st3, i; |
1518 | |
|
1519 | 0 | for(st1=0; stageUTF8Index<stageUTF8Length; ++st1) { |
1520 | 0 | st2=stage1[st1]; |
1521 | 0 | if(st2!=(int32_t)stage1Length/2) { |
1522 | | /* each stage 2 block has 64 entries corresponding to 16 entries in the mbcsIndex */ |
1523 | 0 | for(i=0; i<16; ++i) { |
1524 | 0 | st3=mbcsTable->mbcsIndex[stageUTF8Index++]; |
1525 | 0 | if(st3!=0) { |
1526 | | /* an stage 2 entry's index is per stage 3 16-block, not per stage 3 entry */ |
1527 | 0 | st3>>=4; |
1528 | | /* |
1529 | | * 4 stage 2 entries point to 4 consecutive stage 3 16-blocks which are |
1530 | | * allocated together as a single 64-block for access from the mbcsIndex |
1531 | | */ |
1532 | 0 | stage2[st2++]=st3++; |
1533 | 0 | stage2[st2++]=st3++; |
1534 | 0 | stage2[st2++]=st3++; |
1535 | 0 | stage2[st2++]=st3; |
1536 | 0 | } else { |
1537 | | /* no stage 3 block, skip */ |
1538 | 0 | st2+=4; |
1539 | 0 | } |
1540 | 0 | } |
1541 | 0 | } else { |
1542 | | /* no stage 2 block, skip */ |
1543 | 0 | stageUTF8Index+=16; |
1544 | 0 | } |
1545 | 0 | } |
1546 | 0 | } |
1547 | | |
1548 | | /* reconstitute fromUnicodeBytes with roundtrips from toUnicode data */ |
1549 | 0 | ucnv_MBCSEnumToUnicode(mbcsTable, writeStage3Roundtrip, mbcsTable, pErrorCode); |
1550 | 0 | } |
1551 | | |
1552 | | /* MBCS setup functions ----------------------------------------------------- */ |
1553 | | |
1554 | | static void U_CALLCONV |
1555 | | ucnv_MBCSLoad(UConverterSharedData *sharedData, |
1556 | | UConverterLoadArgs *pArgs, |
1557 | | const uint8_t *raw, |
1558 | 0 | UErrorCode *pErrorCode) { |
1559 | 0 | UDataInfo info; |
1560 | 0 | UConverterMBCSTable *mbcsTable=&sharedData->mbcs; |
1561 | 0 | _MBCSHeader *header=(_MBCSHeader *)raw; |
1562 | 0 | uint32_t offset; |
1563 | 0 | uint32_t headerLength; |
1564 | 0 | UBool noFromU=FALSE; |
1565 | |
|
1566 | 0 | if(header->version[0]==4) { |
1567 | 0 | headerLength=MBCS_HEADER_V4_LENGTH; |
1568 | 0 | } else if(header->version[0]==5 && header->version[1]>=3 && |
1569 | 0 | (header->options&MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK)==0) { |
1570 | 0 | headerLength=header->options&MBCS_OPT_LENGTH_MASK; |
1571 | 0 | noFromU=(UBool)((header->options&MBCS_OPT_NO_FROM_U)!=0); |
1572 | 0 | } else { |
1573 | 0 | *pErrorCode=U_INVALID_TABLE_FORMAT; |
1574 | 0 | return; |
1575 | 0 | } |
1576 | | |
1577 | 0 | mbcsTable->outputType=(uint8_t)header->flags; |
1578 | 0 | if(noFromU && mbcsTable->outputType==MBCS_OUTPUT_1) { |
1579 | 0 | *pErrorCode=U_INVALID_TABLE_FORMAT; |
1580 | 0 | return; |
1581 | 0 | } |
1582 | | |
1583 | | /* extension data, header version 4.2 and higher */ |
1584 | 0 | offset=header->flags>>8; |
1585 | 0 | if(offset!=0) { |
1586 | 0 | mbcsTable->extIndexes=(const int32_t *)(raw+offset); |
1587 | 0 | } |
1588 | |
|
1589 | 0 | if(mbcsTable->outputType==MBCS_OUTPUT_EXT_ONLY) { |
1590 | 0 | UConverterLoadArgs args=UCNV_LOAD_ARGS_INITIALIZER; |
1591 | 0 | UConverterSharedData *baseSharedData; |
1592 | 0 | const int32_t *extIndexes; |
1593 | 0 | const char *baseName; |
1594 | | |
1595 | | /* extension-only file, load the base table and set values appropriately */ |
1596 | 0 | if((extIndexes=mbcsTable->extIndexes)==NULL) { |
1597 | | /* extension-only file without extension */ |
1598 | 0 | *pErrorCode=U_INVALID_TABLE_FORMAT; |
1599 | 0 | return; |
1600 | 0 | } |
1601 | | |
1602 | 0 | if(pArgs->nestedLoads!=1) { |
1603 | | /* an extension table must not be loaded as a base table */ |
1604 | 0 | *pErrorCode=U_INVALID_TABLE_FILE; |
1605 | 0 | return; |
1606 | 0 | } |
1607 | | |
1608 | | /* load the base table */ |
1609 | 0 | baseName=(const char *)header+headerLength*4; |
1610 | 0 | if(0==uprv_strcmp(baseName, sharedData->staticData->name)) { |
1611 | | /* forbid loading this same extension-only file */ |
1612 | 0 | *pErrorCode=U_INVALID_TABLE_FORMAT; |
1613 | 0 | return; |
1614 | 0 | } |
1615 | | |
1616 | | /* TODO parse package name out of the prefix of the base name in the extension .cnv file? */ |
1617 | 0 | args.size=sizeof(UConverterLoadArgs); |
1618 | 0 | args.nestedLoads=2; |
1619 | 0 | args.onlyTestIsLoadable=pArgs->onlyTestIsLoadable; |
1620 | 0 | args.reserved=pArgs->reserved; |
1621 | 0 | args.options=pArgs->options; |
1622 | 0 | args.pkg=pArgs->pkg; |
1623 | 0 | args.name=baseName; |
1624 | 0 | baseSharedData=ucnv_load(&args, pErrorCode); |
1625 | 0 | if(U_FAILURE(*pErrorCode)) { |
1626 | 0 | return; |
1627 | 0 | } |
1628 | 0 | if( baseSharedData->staticData->conversionType!=UCNV_MBCS || |
1629 | 0 | baseSharedData->mbcs.baseSharedData!=NULL |
1630 | 0 | ) { |
1631 | 0 | ucnv_unload(baseSharedData); |
1632 | 0 | *pErrorCode=U_INVALID_TABLE_FORMAT; |
1633 | 0 | return; |
1634 | 0 | } |
1635 | 0 | if(pArgs->onlyTestIsLoadable) { |
1636 | | /* |
1637 | | * Exit as soon as we know that we can load the converter |
1638 | | * and the format is valid and supported. |
1639 | | * The worst that can happen in the following code is a memory |
1640 | | * allocation error. |
1641 | | */ |
1642 | 0 | ucnv_unload(baseSharedData); |
1643 | 0 | return; |
1644 | 0 | } |
1645 | | |
1646 | | /* copy the base table data */ |
1647 | 0 | uprv_memcpy(mbcsTable, &baseSharedData->mbcs, sizeof(UConverterMBCSTable)); |
1648 | | |
1649 | | /* overwrite values with relevant ones for the extension converter */ |
1650 | 0 | mbcsTable->baseSharedData=baseSharedData; |
1651 | 0 | mbcsTable->extIndexes=extIndexes; |
1652 | | |
1653 | | /* |
1654 | | * It would be possible to share the swapLFNL data with a base converter, |
1655 | | * but the generated name would have to be different, and the memory |
1656 | | * would have to be free'd only once. |
1657 | | * It is easier to just create the data for the extension converter |
1658 | | * separately when it is requested. |
1659 | | */ |
1660 | 0 | mbcsTable->swapLFNLStateTable=NULL; |
1661 | 0 | mbcsTable->swapLFNLFromUnicodeBytes=NULL; |
1662 | 0 | mbcsTable->swapLFNLName=NULL; |
1663 | | |
1664 | | /* |
1665 | | * The reconstitutedData must be deleted only when the base converter |
1666 | | * is unloaded. |
1667 | | */ |
1668 | 0 | mbcsTable->reconstitutedData=NULL; |
1669 | | |
1670 | | /* |
1671 | | * Set a special, runtime-only outputType if the extension converter |
1672 | | * is a DBCS version of a base converter that also maps single bytes. |
1673 | | */ |
1674 | 0 | if( sharedData->staticData->conversionType==UCNV_DBCS || |
1675 | 0 | (sharedData->staticData->conversionType==UCNV_MBCS && |
1676 | 0 | sharedData->staticData->minBytesPerChar>=2) |
1677 | 0 | ) { |
1678 | 0 | if(baseSharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO) { |
1679 | | /* the base converter is SI/SO-stateful */ |
1680 | 0 | int32_t entry; |
1681 | | |
1682 | | /* get the dbcs state from the state table entry for SO=0x0e */ |
1683 | 0 | entry=mbcsTable->stateTable[0][0xe]; |
1684 | 0 | if( MBCS_ENTRY_IS_FINAL(entry) && |
1685 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_CHANGE_ONLY && |
1686 | 0 | MBCS_ENTRY_FINAL_STATE(entry)!=0 |
1687 | 0 | ) { |
1688 | 0 | mbcsTable->dbcsOnlyState=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); |
1689 | |
|
1690 | 0 | mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; |
1691 | 0 | } |
1692 | 0 | } else if( |
1693 | 0 | baseSharedData->staticData->conversionType==UCNV_MBCS && |
1694 | 0 | baseSharedData->staticData->minBytesPerChar==1 && |
1695 | 0 | baseSharedData->staticData->maxBytesPerChar==2 && |
1696 | 0 | mbcsTable->countStates<=127 |
1697 | 0 | ) { |
1698 | | /* non-stateful base converter, need to modify the state table */ |
1699 | 0 | int32_t (*newStateTable)[256]; |
1700 | 0 | int32_t *state; |
1701 | 0 | int32_t i, count; |
1702 | | |
1703 | | /* allocate a new state table and copy the base state table contents */ |
1704 | 0 | count=mbcsTable->countStates; |
1705 | 0 | newStateTable=(int32_t (*)[256])uprv_malloc((count+1)*1024); |
1706 | 0 | if(newStateTable==NULL) { |
1707 | 0 | ucnv_unload(baseSharedData); |
1708 | 0 | *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
1709 | 0 | return; |
1710 | 0 | } |
1711 | | |
1712 | 0 | uprv_memcpy(newStateTable, mbcsTable->stateTable, count*1024); |
1713 | | |
1714 | | /* change all final single-byte entries to go to a new all-illegal state */ |
1715 | 0 | state=newStateTable[0]; |
1716 | 0 | for(i=0; i<256; ++i) { |
1717 | 0 | if(MBCS_ENTRY_IS_FINAL(state[i])) { |
1718 | 0 | state[i]=MBCS_ENTRY_TRANSITION(count, 0); |
1719 | 0 | } |
1720 | 0 | } |
1721 | | |
1722 | | /* build the new all-illegal state */ |
1723 | 0 | state=newStateTable[count]; |
1724 | 0 | for(i=0; i<256; ++i) { |
1725 | 0 | state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0); |
1726 | 0 | } |
1727 | 0 | mbcsTable->stateTable=(const int32_t (*)[256])newStateTable; |
1728 | 0 | mbcsTable->countStates=(uint8_t)(count+1); |
1729 | 0 | mbcsTable->stateTableOwned=TRUE; |
1730 | |
|
1731 | 0 | mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; |
1732 | 0 | } |
1733 | 0 | } |
1734 | | |
1735 | | /* |
1736 | | * unlike below for files with base tables, do not get the unicodeMask |
1737 | | * from the sharedData; instead, use the base table's unicodeMask, |
1738 | | * which we copied in the memcpy above; |
1739 | | * this is necessary because the static data unicodeMask, especially |
1740 | | * the UCNV_HAS_SUPPLEMENTARY flag, is part of the base table data |
1741 | | */ |
1742 | 0 | } else { |
1743 | | /* conversion file with a base table; an additional extension table is optional */ |
1744 | | /* make sure that the output type is known */ |
1745 | 0 | switch(mbcsTable->outputType) { |
1746 | 0 | case MBCS_OUTPUT_1: |
1747 | 0 | case MBCS_OUTPUT_2: |
1748 | 0 | case MBCS_OUTPUT_3: |
1749 | 0 | case MBCS_OUTPUT_4: |
1750 | 0 | case MBCS_OUTPUT_3_EUC: |
1751 | 0 | case MBCS_OUTPUT_4_EUC: |
1752 | 0 | case MBCS_OUTPUT_2_SISO: |
1753 | | /* OK */ |
1754 | 0 | break; |
1755 | 0 | default: |
1756 | 0 | *pErrorCode=U_INVALID_TABLE_FORMAT; |
1757 | 0 | return; |
1758 | 0 | } |
1759 | 0 | if(pArgs->onlyTestIsLoadable) { |
1760 | | /* |
1761 | | * Exit as soon as we know that we can load the converter |
1762 | | * and the format is valid and supported. |
1763 | | * The worst that can happen in the following code is a memory |
1764 | | * allocation error. |
1765 | | */ |
1766 | 0 | return; |
1767 | 0 | } |
1768 | | |
1769 | 0 | mbcsTable->countStates=(uint8_t)header->countStates; |
1770 | 0 | mbcsTable->countToUFallbacks=header->countToUFallbacks; |
1771 | 0 | mbcsTable->stateTable=(const int32_t (*)[256])(raw+headerLength*4); |
1772 | 0 | mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates); |
1773 | 0 | mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits); |
1774 | |
|
1775 | 0 | mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable); |
1776 | 0 | mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes); |
1777 | 0 | mbcsTable->fromUBytesLength=header->fromUBytesLength; |
1778 | | |
1779 | | /* |
1780 | | * converter versions 6.1 and up contain a unicodeMask that is |
1781 | | * used here to select the most efficient function implementations |
1782 | | */ |
1783 | 0 | info.size=sizeof(UDataInfo); |
1784 | 0 | udata_getInfo((UDataMemory *)sharedData->dataMemory, &info); |
1785 | 0 | if(info.formatVersion[0]>6 || (info.formatVersion[0]==6 && info.formatVersion[1]>=1)) { |
1786 | | /* mask off possible future extensions to be safe */ |
1787 | 0 | mbcsTable->unicodeMask=(uint8_t)(sharedData->staticData->unicodeMask&3); |
1788 | 0 | } else { |
1789 | | /* for older versions, assume worst case: contains anything possible (prevent over-optimizations) */ |
1790 | 0 | mbcsTable->unicodeMask=UCNV_HAS_SUPPLEMENTARY|UCNV_HAS_SURROGATES; |
1791 | 0 | } |
1792 | | |
1793 | | /* |
1794 | | * _MBCSHeader.version 4.3 adds utf8Friendly data structures. |
1795 | | * Check for the header version, SBCS vs. MBCS, and for whether the |
1796 | | * data structures are optimized for code points as high as what the |
1797 | | * runtime code is designed for. |
1798 | | * The implementation does not handle mapping tables with entries for |
1799 | | * unpaired surrogates. |
1800 | | */ |
1801 | 0 | if( header->version[1]>=3 && |
1802 | 0 | (mbcsTable->unicodeMask&UCNV_HAS_SURROGATES)==0 && |
1803 | 0 | (mbcsTable->countStates==1 ? |
1804 | 0 | (header->version[2]>=(SBCS_FAST_MAX>>8)) : |
1805 | 0 | (header->version[2]>=(MBCS_FAST_MAX>>8)) |
1806 | 0 | ) |
1807 | 0 | ) { |
1808 | 0 | mbcsTable->utf8Friendly=TRUE; |
1809 | |
|
1810 | 0 | if(mbcsTable->countStates==1) { |
1811 | | /* |
1812 | | * SBCS: Stage 3 is allocated in 64-entry blocks for U+0000..SBCS_FAST_MAX or higher. |
1813 | | * Build a table with indexes to each block, to be used instead of |
1814 | | * the regular stage 1/2 table. |
1815 | | */ |
1816 | 0 | int32_t i; |
1817 | 0 | for(i=0; i<(SBCS_FAST_LIMIT>>6); ++i) { |
1818 | 0 | mbcsTable->sbcsIndex[i]=mbcsTable->fromUnicodeTable[mbcsTable->fromUnicodeTable[i>>4]+((i<<2)&0x3c)]; |
1819 | 0 | } |
1820 | | /* set SBCS_FAST_MAX to reflect the reach of sbcsIndex[] even if header->version[2]>(SBCS_FAST_MAX>>8) */ |
1821 | 0 | mbcsTable->maxFastUChar=SBCS_FAST_MAX; |
1822 | 0 | } else { |
1823 | | /* |
1824 | | * MBCS: Stage 3 is allocated in 64-entry blocks for U+0000..MBCS_FAST_MAX or higher. |
1825 | | * The .cnv file is prebuilt with an additional stage table with indexes |
1826 | | * to each block. |
1827 | | */ |
1828 | 0 | mbcsTable->mbcsIndex=(const uint16_t *) |
1829 | 0 | (mbcsTable->fromUnicodeBytes+ |
1830 | 0 | (noFromU ? 0 : mbcsTable->fromUBytesLength)); |
1831 | 0 | mbcsTable->maxFastUChar=(((UChar)header->version[2])<<8)|0xff; |
1832 | 0 | } |
1833 | 0 | } |
1834 | | |
1835 | | /* calculate a bit set of 4 ASCII characters per bit that round-trip to ASCII bytes */ |
1836 | 0 | { |
1837 | 0 | uint32_t asciiRoundtrips=0xffffffff; |
1838 | 0 | int32_t i; |
1839 | |
|
1840 | 0 | for(i=0; i<0x80; ++i) { |
1841 | 0 | if(mbcsTable->stateTable[0][i]!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, i)) { |
1842 | 0 | asciiRoundtrips&=~((uint32_t)1<<(i>>2)); |
1843 | 0 | } |
1844 | 0 | } |
1845 | 0 | mbcsTable->asciiRoundtrips=asciiRoundtrips; |
1846 | 0 | } |
1847 | |
|
1848 | 0 | if(noFromU) { |
1849 | 0 | uint32_t stage1Length= |
1850 | 0 | mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY ? |
1851 | 0 | 0x440 : 0x40; |
1852 | 0 | uint32_t stage2Length= |
1853 | 0 | (header->offsetFromUBytes-header->offsetFromUTable)/4- |
1854 | 0 | stage1Length/2; |
1855 | 0 | reconstituteData(mbcsTable, stage1Length, stage2Length, header->fullStage2Length, pErrorCode); |
1856 | 0 | } |
1857 | 0 | } |
1858 | | |
1859 | | /* Set the impl pointer here so that it is set for both extension-only and base tables. */ |
1860 | 0 | if(mbcsTable->utf8Friendly) { |
1861 | 0 | if(mbcsTable->countStates==1) { |
1862 | 0 | sharedData->impl=&_SBCSUTF8Impl; |
1863 | 0 | } else { |
1864 | 0 | if(mbcsTable->outputType==MBCS_OUTPUT_2) { |
1865 | 0 | sharedData->impl=&_DBCSUTF8Impl; |
1866 | 0 | } |
1867 | 0 | } |
1868 | 0 | } |
1869 | |
|
1870 | 0 | if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) { |
1871 | | /* |
1872 | | * MBCS_OUTPUT_DBCS_ONLY: No SBCS mappings, therefore ASCII does not roundtrip. |
1873 | | * MBCS_OUTPUT_2_SISO: Bypass the ASCII fastpath to handle prevLength correctly. |
1874 | | */ |
1875 | 0 | mbcsTable->asciiRoundtrips=0; |
1876 | 0 | } |
1877 | 0 | } |
1878 | | |
1879 | | static void U_CALLCONV |
1880 | 0 | ucnv_MBCSUnload(UConverterSharedData *sharedData) { |
1881 | 0 | UConverterMBCSTable *mbcsTable=&sharedData->mbcs; |
1882 | |
|
1883 | 0 | if(mbcsTable->swapLFNLStateTable!=NULL) { |
1884 | 0 | uprv_free(mbcsTable->swapLFNLStateTable); |
1885 | 0 | } |
1886 | 0 | if(mbcsTable->stateTableOwned) { |
1887 | 0 | uprv_free((void *)mbcsTable->stateTable); |
1888 | 0 | } |
1889 | 0 | if(mbcsTable->baseSharedData!=NULL) { |
1890 | 0 | ucnv_unload(mbcsTable->baseSharedData); |
1891 | 0 | } |
1892 | 0 | if(mbcsTable->reconstitutedData!=NULL) { |
1893 | 0 | uprv_free(mbcsTable->reconstitutedData); |
1894 | 0 | } |
1895 | 0 | } |
1896 | | |
1897 | | static void U_CALLCONV |
1898 | | ucnv_MBCSOpen(UConverter *cnv, |
1899 | | UConverterLoadArgs *pArgs, |
1900 | 0 | UErrorCode *pErrorCode) { |
1901 | 0 | UConverterMBCSTable *mbcsTable; |
1902 | 0 | const int32_t *extIndexes; |
1903 | 0 | uint8_t outputType; |
1904 | 0 | int8_t maxBytesPerUChar; |
1905 | |
|
1906 | 0 | if(pArgs->onlyTestIsLoadable) { |
1907 | 0 | return; |
1908 | 0 | } |
1909 | | |
1910 | 0 | mbcsTable=&cnv->sharedData->mbcs; |
1911 | 0 | outputType=mbcsTable->outputType; |
1912 | |
|
1913 | 0 | if(outputType==MBCS_OUTPUT_DBCS_ONLY) { |
1914 | | /* the swaplfnl option does not apply, remove it */ |
1915 | 0 | cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL; |
1916 | 0 | } |
1917 | |
|
1918 | 0 | if((pArgs->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
1919 | | /* do this because double-checked locking is broken */ |
1920 | 0 | UBool isCached; |
1921 | |
|
1922 | 0 | icu::umtx_lock(NULL); |
1923 | 0 | isCached=mbcsTable->swapLFNLStateTable!=NULL; |
1924 | 0 | icu::umtx_unlock(NULL); |
1925 | |
|
1926 | 0 | if(!isCached) { |
1927 | 0 | if(!_EBCDICSwapLFNL(cnv->sharedData, pErrorCode)) { |
1928 | 0 | if(U_FAILURE(*pErrorCode)) { |
1929 | 0 | return; /* something went wrong */ |
1930 | 0 | } |
1931 | | |
1932 | | /* the option does not apply, remove it */ |
1933 | 0 | cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL; |
1934 | 0 | } |
1935 | 0 | } |
1936 | 0 | } |
1937 | | |
1938 | 0 | if(uprv_strstr(pArgs->name, "18030")!=NULL) { |
1939 | 0 | if(uprv_strstr(pArgs->name, "gb18030")!=NULL || uprv_strstr(pArgs->name, "GB18030")!=NULL) { |
1940 | | /* set a flag for GB 18030 mode, which changes the callback behavior */ |
1941 | 0 | cnv->options|=_MBCS_OPTION_GB18030; |
1942 | 0 | } |
1943 | 0 | } else if((uprv_strstr(pArgs->name, "KEIS")!=NULL) || (uprv_strstr(pArgs->name, "keis")!=NULL)) { |
1944 | | /* set a flag for KEIS converter, which changes the SI/SO character sequence */ |
1945 | 0 | cnv->options|=_MBCS_OPTION_KEIS; |
1946 | 0 | } else if((uprv_strstr(pArgs->name, "JEF")!=NULL) || (uprv_strstr(pArgs->name, "jef")!=NULL)) { |
1947 | | /* set a flag for JEF converter, which changes the SI/SO character sequence */ |
1948 | 0 | cnv->options|=_MBCS_OPTION_JEF; |
1949 | 0 | } else if((uprv_strstr(pArgs->name, "JIPS")!=NULL) || (uprv_strstr(pArgs->name, "jips")!=NULL)) { |
1950 | | /* set a flag for JIPS converter, which changes the SI/SO character sequence */ |
1951 | 0 | cnv->options|=_MBCS_OPTION_JIPS; |
1952 | 0 | } |
1953 | | |
1954 | | /* fix maxBytesPerUChar depending on outputType and options etc. */ |
1955 | 0 | if(outputType==MBCS_OUTPUT_2_SISO) { |
1956 | 0 | cnv->maxBytesPerUChar=3; /* SO+DBCS */ |
1957 | 0 | } |
1958 | |
|
1959 | 0 | extIndexes=mbcsTable->extIndexes; |
1960 | 0 | if(extIndexes!=NULL) { |
1961 | 0 | maxBytesPerUChar=(int8_t)UCNV_GET_MAX_BYTES_PER_UCHAR(extIndexes); |
1962 | 0 | if(outputType==MBCS_OUTPUT_2_SISO) { |
1963 | 0 | ++maxBytesPerUChar; /* SO + multiple DBCS */ |
1964 | 0 | } |
1965 | |
|
1966 | 0 | if(maxBytesPerUChar>cnv->maxBytesPerUChar) { |
1967 | 0 | cnv->maxBytesPerUChar=maxBytesPerUChar; |
1968 | 0 | } |
1969 | 0 | } |
1970 | |
|
1971 | | #if 0 |
1972 | | /* |
1973 | | * documentation of UConverter fields used for status |
1974 | | * all of these fields are (re)set to 0 by ucnv_bld.c and ucnv_reset() |
1975 | | */ |
1976 | | |
1977 | | /* toUnicode */ |
1978 | | cnv->toUnicodeStatus=0; /* offset */ |
1979 | | cnv->mode=0; /* state */ |
1980 | | cnv->toULength=0; /* byteIndex */ |
1981 | | |
1982 | | /* fromUnicode */ |
1983 | | cnv->fromUChar32=0; |
1984 | | cnv->fromUnicodeStatus=1; /* prevLength */ |
1985 | | #endif |
1986 | 0 | } |
1987 | | |
1988 | | U_CDECL_BEGIN |
1989 | | |
1990 | | static const char* U_CALLCONV |
1991 | 0 | ucnv_MBCSGetName(const UConverter *cnv) { |
1992 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0 && cnv->sharedData->mbcs.swapLFNLName!=NULL) { |
1993 | 0 | return cnv->sharedData->mbcs.swapLFNLName; |
1994 | 0 | } else { |
1995 | 0 | return cnv->sharedData->staticData->name; |
1996 | 0 | } |
1997 | 0 | } |
1998 | | U_CDECL_END |
1999 | | |
2000 | | |
2001 | | /* MBCS-to-Unicode conversion functions ------------------------------------- */ |
2002 | | |
2003 | | static UChar32 U_CALLCONV |
2004 | 0 | ucnv_MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) { |
2005 | 0 | const _MBCSToUFallback *toUFallbacks; |
2006 | 0 | uint32_t i, start, limit; |
2007 | |
|
2008 | 0 | limit=mbcsTable->countToUFallbacks; |
2009 | 0 | if(limit>0) { |
2010 | | /* do a binary search for the fallback mapping */ |
2011 | 0 | toUFallbacks=mbcsTable->toUFallbacks; |
2012 | 0 | start=0; |
2013 | 0 | while(start<limit-1) { |
2014 | 0 | i=(start+limit)/2; |
2015 | 0 | if(offset<toUFallbacks[i].offset) { |
2016 | 0 | limit=i; |
2017 | 0 | } else { |
2018 | 0 | start=i; |
2019 | 0 | } |
2020 | 0 | } |
2021 | | |
2022 | | /* did we really find it? */ |
2023 | 0 | if(offset==toUFallbacks[start].offset) { |
2024 | 0 | return toUFallbacks[start].codePoint; |
2025 | 0 | } |
2026 | 0 | } |
2027 | | |
2028 | 0 | return 0xfffe; |
2029 | 0 | } |
2030 | | |
2031 | | /* This version of ucnv_MBCSToUnicodeWithOffsets() is optimized for single-byte, single-state codepages. */ |
2032 | | static void |
2033 | | ucnv_MBCSSingleToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, |
2034 | 0 | UErrorCode *pErrorCode) { |
2035 | 0 | UConverter *cnv; |
2036 | 0 | const uint8_t *source, *sourceLimit; |
2037 | 0 | UChar *target; |
2038 | 0 | const UChar *targetLimit; |
2039 | 0 | int32_t *offsets; |
2040 | |
|
2041 | 0 | const int32_t (*stateTable)[256]; |
2042 | |
|
2043 | 0 | int32_t sourceIndex; |
2044 | |
|
2045 | 0 | int32_t entry; |
2046 | 0 | UChar c; |
2047 | 0 | uint8_t action; |
2048 | | |
2049 | | /* set up the local pointers */ |
2050 | 0 | cnv=pArgs->converter; |
2051 | 0 | source=(const uint8_t *)pArgs->source; |
2052 | 0 | sourceLimit=(const uint8_t *)pArgs->sourceLimit; |
2053 | 0 | target=pArgs->target; |
2054 | 0 | targetLimit=pArgs->targetLimit; |
2055 | 0 | offsets=pArgs->offsets; |
2056 | |
|
2057 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
2058 | 0 | stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; |
2059 | 0 | } else { |
2060 | 0 | stateTable=cnv->sharedData->mbcs.stateTable; |
2061 | 0 | } |
2062 | | |
2063 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
2064 | 0 | sourceIndex=0; |
2065 | | |
2066 | | /* conversion loop */ |
2067 | 0 | while(source<sourceLimit) { |
2068 | | /* |
2069 | | * This following test is to see if available input would overflow the output. |
2070 | | * It does not catch output of more than one code unit that |
2071 | | * overflows as a result of a surrogate pair or callback output |
2072 | | * from the last source byte. |
2073 | | * Therefore, those situations also test for overflows and will |
2074 | | * then break the loop, too. |
2075 | | */ |
2076 | 0 | if(target>=targetLimit) { |
2077 | | /* target is full */ |
2078 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2079 | 0 | break; |
2080 | 0 | } |
2081 | | |
2082 | 0 | entry=stateTable[0][*source++]; |
2083 | | /* MBCS_ENTRY_IS_FINAL(entry) */ |
2084 | | |
2085 | | /* test the most common case first */ |
2086 | 0 | if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { |
2087 | | /* output BMP code point */ |
2088 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2089 | 0 | if(offsets!=NULL) { |
2090 | 0 | *offsets++=sourceIndex; |
2091 | 0 | } |
2092 | | |
2093 | | /* normal end of action codes: prepare for a new character */ |
2094 | 0 | ++sourceIndex; |
2095 | 0 | continue; |
2096 | 0 | } |
2097 | | |
2098 | | /* |
2099 | | * An if-else-if chain provides more reliable performance for |
2100 | | * the most common cases compared to a switch. |
2101 | | */ |
2102 | 0 | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
2103 | 0 | if(action==MBCS_STATE_VALID_DIRECT_20 || |
2104 | 0 | (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) |
2105 | 0 | ) { |
2106 | 0 | entry=MBCS_ENTRY_FINAL_VALUE(entry); |
2107 | | /* output surrogate pair */ |
2108 | 0 | *target++=(UChar)(0xd800|(UChar)(entry>>10)); |
2109 | 0 | if(offsets!=NULL) { |
2110 | 0 | *offsets++=sourceIndex; |
2111 | 0 | } |
2112 | 0 | c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); |
2113 | 0 | if(target<targetLimit) { |
2114 | 0 | *target++=c; |
2115 | 0 | if(offsets!=NULL) { |
2116 | 0 | *offsets++=sourceIndex; |
2117 | 0 | } |
2118 | 0 | } else { |
2119 | | /* target overflow */ |
2120 | 0 | cnv->UCharErrorBuffer[0]=c; |
2121 | 0 | cnv->UCharErrorBufferLength=1; |
2122 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2123 | 0 | break; |
2124 | 0 | } |
2125 | | |
2126 | 0 | ++sourceIndex; |
2127 | 0 | continue; |
2128 | 0 | } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
2129 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv)) { |
2130 | | /* output BMP code point */ |
2131 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2132 | 0 | if(offsets!=NULL) { |
2133 | 0 | *offsets++=sourceIndex; |
2134 | 0 | } |
2135 | |
|
2136 | 0 | ++sourceIndex; |
2137 | 0 | continue; |
2138 | 0 | } |
2139 | 0 | } else if(action==MBCS_STATE_UNASSIGNED) { |
2140 | | /* just fall through */ |
2141 | 0 | } else if(action==MBCS_STATE_ILLEGAL) { |
2142 | | /* callback(illegal) */ |
2143 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2144 | 0 | } else { |
2145 | | /* reserved, must never occur */ |
2146 | 0 | ++sourceIndex; |
2147 | 0 | continue; |
2148 | 0 | } |
2149 | | |
2150 | 0 | if(U_FAILURE(*pErrorCode)) { |
2151 | | /* callback(illegal) */ |
2152 | 0 | break; |
2153 | 0 | } else /* unassigned sequences indicated with byteIndex>0 */ { |
2154 | | /* try an extension mapping */ |
2155 | 0 | pArgs->source=(const char *)source; |
2156 | 0 | cnv->toUBytes[0]=*(source-1); |
2157 | 0 | cnv->toULength=_extToU(cnv, cnv->sharedData, |
2158 | 0 | 1, &source, sourceLimit, |
2159 | 0 | &target, targetLimit, |
2160 | 0 | &offsets, sourceIndex, |
2161 | 0 | pArgs->flush, |
2162 | 0 | pErrorCode); |
2163 | 0 | sourceIndex+=1+(int32_t)(source-(const uint8_t *)pArgs->source); |
2164 | |
|
2165 | 0 | if(U_FAILURE(*pErrorCode)) { |
2166 | | /* not mappable or buffer overflow */ |
2167 | 0 | break; |
2168 | 0 | } |
2169 | 0 | } |
2170 | 0 | } |
2171 | | |
2172 | | /* write back the updated pointers */ |
2173 | 0 | pArgs->source=(const char *)source; |
2174 | 0 | pArgs->target=target; |
2175 | 0 | pArgs->offsets=offsets; |
2176 | 0 | } |
2177 | | |
2178 | | /* |
2179 | | * This version of ucnv_MBCSSingleToUnicodeWithOffsets() is optimized for single-byte, single-state codepages |
2180 | | * that only map to and from the BMP. |
2181 | | * In addition to single-byte optimizations, the offset calculations |
2182 | | * become much easier. |
2183 | | */ |
2184 | | static void |
2185 | | ucnv_MBCSSingleToBMPWithOffsets(UConverterToUnicodeArgs *pArgs, |
2186 | 0 | UErrorCode *pErrorCode) { |
2187 | 0 | UConverter *cnv; |
2188 | 0 | const uint8_t *source, *sourceLimit, *lastSource; |
2189 | 0 | UChar *target; |
2190 | 0 | int32_t targetCapacity, length; |
2191 | 0 | int32_t *offsets; |
2192 | |
|
2193 | 0 | const int32_t (*stateTable)[256]; |
2194 | |
|
2195 | 0 | int32_t sourceIndex; |
2196 | |
|
2197 | 0 | int32_t entry; |
2198 | 0 | uint8_t action; |
2199 | | |
2200 | | /* set up the local pointers */ |
2201 | 0 | cnv=pArgs->converter; |
2202 | 0 | source=(const uint8_t *)pArgs->source; |
2203 | 0 | sourceLimit=(const uint8_t *)pArgs->sourceLimit; |
2204 | 0 | target=pArgs->target; |
2205 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); |
2206 | 0 | offsets=pArgs->offsets; |
2207 | |
|
2208 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
2209 | 0 | stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; |
2210 | 0 | } else { |
2211 | 0 | stateTable=cnv->sharedData->mbcs.stateTable; |
2212 | 0 | } |
2213 | | |
2214 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
2215 | 0 | sourceIndex=0; |
2216 | 0 | lastSource=source; |
2217 | | |
2218 | | /* |
2219 | | * since the conversion here is 1:1 UChar:uint8_t, we need only one counter |
2220 | | * for the minimum of the sourceLength and targetCapacity |
2221 | | */ |
2222 | 0 | length=(int32_t)(sourceLimit-source); |
2223 | 0 | if(length<targetCapacity) { |
2224 | 0 | targetCapacity=length; |
2225 | 0 | } |
2226 | |
|
2227 | 0 | #if MBCS_UNROLL_SINGLE_TO_BMP |
2228 | | /* unrolling makes it faster on Pentium III/Windows 2000 */ |
2229 | | /* unroll the loop with the most common case */ |
2230 | 0 | unrolled: |
2231 | 0 | if(targetCapacity>=16) { |
2232 | 0 | int32_t count, loops, oredEntries; |
2233 | |
|
2234 | 0 | loops=count=targetCapacity>>4; |
2235 | 0 | do { |
2236 | 0 | oredEntries=entry=stateTable[0][*source++]; |
2237 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2238 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2239 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2240 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2241 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2242 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2243 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2244 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2245 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2246 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2247 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2248 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2249 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2250 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2251 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2252 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2253 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2254 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2255 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2256 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2257 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2258 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2259 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2260 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2261 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2262 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2263 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2264 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2265 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2266 | 0 | oredEntries|=entry=stateTable[0][*source++]; |
2267 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2268 | | |
2269 | | /* were all 16 entries really valid? */ |
2270 | 0 | if(!MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(oredEntries)) { |
2271 | | /* no, return to the first of these 16 */ |
2272 | 0 | source-=16; |
2273 | 0 | target-=16; |
2274 | 0 | break; |
2275 | 0 | } |
2276 | 0 | } while(--count>0); |
2277 | 0 | count=loops-count; |
2278 | 0 | targetCapacity-=16*count; |
2279 | |
|
2280 | 0 | if(offsets!=NULL) { |
2281 | 0 | lastSource+=16*count; |
2282 | 0 | while(count>0) { |
2283 | 0 | *offsets++=sourceIndex++; |
2284 | 0 | *offsets++=sourceIndex++; |
2285 | 0 | *offsets++=sourceIndex++; |
2286 | 0 | *offsets++=sourceIndex++; |
2287 | 0 | *offsets++=sourceIndex++; |
2288 | 0 | *offsets++=sourceIndex++; |
2289 | 0 | *offsets++=sourceIndex++; |
2290 | 0 | *offsets++=sourceIndex++; |
2291 | 0 | *offsets++=sourceIndex++; |
2292 | 0 | *offsets++=sourceIndex++; |
2293 | 0 | *offsets++=sourceIndex++; |
2294 | 0 | *offsets++=sourceIndex++; |
2295 | 0 | *offsets++=sourceIndex++; |
2296 | 0 | *offsets++=sourceIndex++; |
2297 | 0 | *offsets++=sourceIndex++; |
2298 | 0 | *offsets++=sourceIndex++; |
2299 | 0 | --count; |
2300 | 0 | } |
2301 | 0 | } |
2302 | 0 | } |
2303 | 0 | #endif |
2304 | | |
2305 | | /* conversion loop */ |
2306 | 0 | while(targetCapacity > 0 && source < sourceLimit) { |
2307 | 0 | entry=stateTable[0][*source++]; |
2308 | | /* MBCS_ENTRY_IS_FINAL(entry) */ |
2309 | | |
2310 | | /* test the most common case first */ |
2311 | 0 | if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { |
2312 | | /* output BMP code point */ |
2313 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2314 | 0 | --targetCapacity; |
2315 | 0 | continue; |
2316 | 0 | } |
2317 | | |
2318 | | /* |
2319 | | * An if-else-if chain provides more reliable performance for |
2320 | | * the most common cases compared to a switch. |
2321 | | */ |
2322 | 0 | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
2323 | 0 | if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
2324 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv)) { |
2325 | | /* output BMP code point */ |
2326 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2327 | 0 | --targetCapacity; |
2328 | 0 | continue; |
2329 | 0 | } |
2330 | 0 | } else if(action==MBCS_STATE_UNASSIGNED) { |
2331 | | /* just fall through */ |
2332 | 0 | } else if(action==MBCS_STATE_ILLEGAL) { |
2333 | | /* callback(illegal) */ |
2334 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2335 | 0 | } else { |
2336 | | /* reserved, must never occur */ |
2337 | 0 | continue; |
2338 | 0 | } |
2339 | | |
2340 | | /* set offsets since the start or the last extension */ |
2341 | 0 | if(offsets!=NULL) { |
2342 | 0 | int32_t count=(int32_t)(source-lastSource); |
2343 | | |
2344 | | /* predecrement: do not set the offset for the callback-causing character */ |
2345 | 0 | while(--count>0) { |
2346 | 0 | *offsets++=sourceIndex++; |
2347 | 0 | } |
2348 | | /* offset and sourceIndex are now set for the current character */ |
2349 | 0 | } |
2350 | |
|
2351 | 0 | if(U_FAILURE(*pErrorCode)) { |
2352 | | /* callback(illegal) */ |
2353 | 0 | break; |
2354 | 0 | } else /* unassigned sequences indicated with byteIndex>0 */ { |
2355 | | /* try an extension mapping */ |
2356 | 0 | lastSource=source; |
2357 | 0 | cnv->toUBytes[0]=*(source-1); |
2358 | 0 | cnv->toULength=_extToU(cnv, cnv->sharedData, |
2359 | 0 | 1, &source, sourceLimit, |
2360 | 0 | &target, pArgs->targetLimit, |
2361 | 0 | &offsets, sourceIndex, |
2362 | 0 | pArgs->flush, |
2363 | 0 | pErrorCode); |
2364 | 0 | sourceIndex+=1+(int32_t)(source-lastSource); |
2365 | |
|
2366 | 0 | if(U_FAILURE(*pErrorCode)) { |
2367 | | /* not mappable or buffer overflow */ |
2368 | 0 | break; |
2369 | 0 | } |
2370 | | |
2371 | | /* recalculate the targetCapacity after an extension mapping */ |
2372 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-target); |
2373 | 0 | length=(int32_t)(sourceLimit-source); |
2374 | 0 | if(length<targetCapacity) { |
2375 | 0 | targetCapacity=length; |
2376 | 0 | } |
2377 | 0 | } |
2378 | | |
2379 | 0 | #if MBCS_UNROLL_SINGLE_TO_BMP |
2380 | | /* unrolling makes it faster on Pentium III/Windows 2000 */ |
2381 | 0 | goto unrolled; |
2382 | 0 | #endif |
2383 | 0 | } |
2384 | | |
2385 | 0 | if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=pArgs->targetLimit) { |
2386 | | /* target is full */ |
2387 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2388 | 0 | } |
2389 | | |
2390 | | /* set offsets since the start or the last callback */ |
2391 | 0 | if(offsets!=NULL) { |
2392 | 0 | size_t count=source-lastSource; |
2393 | 0 | while(count>0) { |
2394 | 0 | *offsets++=sourceIndex++; |
2395 | 0 | --count; |
2396 | 0 | } |
2397 | 0 | } |
2398 | | |
2399 | | /* write back the updated pointers */ |
2400 | 0 | pArgs->source=(const char *)source; |
2401 | 0 | pArgs->target=target; |
2402 | 0 | pArgs->offsets=offsets; |
2403 | 0 | } |
2404 | | |
2405 | | static UBool |
2406 | 0 | hasValidTrailBytes(const int32_t (*stateTable)[256], uint8_t state) { |
2407 | 0 | const int32_t *row=stateTable[state]; |
2408 | 0 | int32_t b, entry; |
2409 | | /* First test for final entries in this state for some commonly valid byte values. */ |
2410 | 0 | entry=row[0xa1]; |
2411 | 0 | if( !MBCS_ENTRY_IS_TRANSITION(entry) && |
2412 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL |
2413 | 0 | ) { |
2414 | 0 | return TRUE; |
2415 | 0 | } |
2416 | 0 | entry=row[0x41]; |
2417 | 0 | if( !MBCS_ENTRY_IS_TRANSITION(entry) && |
2418 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL |
2419 | 0 | ) { |
2420 | 0 | return TRUE; |
2421 | 0 | } |
2422 | | /* Then test for final entries in this state. */ |
2423 | 0 | for(b=0; b<=0xff; ++b) { |
2424 | 0 | entry=row[b]; |
2425 | 0 | if( !MBCS_ENTRY_IS_TRANSITION(entry) && |
2426 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL |
2427 | 0 | ) { |
2428 | 0 | return TRUE; |
2429 | 0 | } |
2430 | 0 | } |
2431 | | /* Then recurse for transition entries. */ |
2432 | 0 | for(b=0; b<=0xff; ++b) { |
2433 | 0 | entry=row[b]; |
2434 | 0 | if( MBCS_ENTRY_IS_TRANSITION(entry) && |
2435 | 0 | hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry)) |
2436 | 0 | ) { |
2437 | 0 | return TRUE; |
2438 | 0 | } |
2439 | 0 | } |
2440 | 0 | return FALSE; |
2441 | 0 | } |
2442 | | |
2443 | | /* |
2444 | | * Is byte b a single/lead byte in this state? |
2445 | | * Recurse for transition states, because here we don't want to say that |
2446 | | * b is a lead byte if all byte sequences that start with b are illegal. |
2447 | | */ |
2448 | | static UBool |
2449 | 0 | isSingleOrLead(const int32_t (*stateTable)[256], uint8_t state, UBool isDBCSOnly, uint8_t b) { |
2450 | 0 | const int32_t *row=stateTable[state]; |
2451 | 0 | int32_t entry=row[b]; |
2452 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { /* lead byte */ |
2453 | 0 | return hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry)); |
2454 | 0 | } else { |
2455 | 0 | uint8_t action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
2456 | 0 | if(action==MBCS_STATE_CHANGE_ONLY && isDBCSOnly) { |
2457 | 0 | return FALSE; /* SI/SO are illegal for DBCS-only conversion */ |
2458 | 0 | } else { |
2459 | 0 | return action!=MBCS_STATE_ILLEGAL; |
2460 | 0 | } |
2461 | 0 | } |
2462 | 0 | } |
2463 | | |
2464 | | U_CFUNC void |
2465 | | ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, |
2466 | 0 | UErrorCode *pErrorCode) { |
2467 | 0 | UConverter *cnv; |
2468 | 0 | const uint8_t *source, *sourceLimit; |
2469 | 0 | UChar *target; |
2470 | 0 | const UChar *targetLimit; |
2471 | 0 | int32_t *offsets; |
2472 | |
|
2473 | 0 | const int32_t (*stateTable)[256]; |
2474 | 0 | const uint16_t *unicodeCodeUnits; |
2475 | |
|
2476 | 0 | uint32_t offset; |
2477 | 0 | uint8_t state; |
2478 | 0 | int8_t byteIndex; |
2479 | 0 | uint8_t *bytes; |
2480 | |
|
2481 | 0 | int32_t sourceIndex, nextSourceIndex; |
2482 | |
|
2483 | 0 | int32_t entry; |
2484 | 0 | UChar c; |
2485 | 0 | uint8_t action; |
2486 | | |
2487 | | /* use optimized function if possible */ |
2488 | 0 | cnv=pArgs->converter; |
2489 | |
|
2490 | 0 | if(cnv->preToULength>0) { |
2491 | | /* |
2492 | | * pass sourceIndex=-1 because we continue from an earlier buffer |
2493 | | * in the future, this may change with continuous offsets |
2494 | | */ |
2495 | 0 | ucnv_extContinueMatchToU(cnv, pArgs, -1, pErrorCode); |
2496 | |
|
2497 | 0 | if(U_FAILURE(*pErrorCode) || cnv->preToULength<0) { |
2498 | 0 | return; |
2499 | 0 | } |
2500 | 0 | } |
2501 | | |
2502 | 0 | if(cnv->sharedData->mbcs.countStates==1) { |
2503 | 0 | if(!(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { |
2504 | 0 | ucnv_MBCSSingleToBMPWithOffsets(pArgs, pErrorCode); |
2505 | 0 | } else { |
2506 | 0 | ucnv_MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode); |
2507 | 0 | } |
2508 | 0 | return; |
2509 | 0 | } |
2510 | | |
2511 | | /* set up the local pointers */ |
2512 | 0 | source=(const uint8_t *)pArgs->source; |
2513 | 0 | sourceLimit=(const uint8_t *)pArgs->sourceLimit; |
2514 | 0 | target=pArgs->target; |
2515 | 0 | targetLimit=pArgs->targetLimit; |
2516 | 0 | offsets=pArgs->offsets; |
2517 | |
|
2518 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
2519 | 0 | stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; |
2520 | 0 | } else { |
2521 | 0 | stateTable=cnv->sharedData->mbcs.stateTable; |
2522 | 0 | } |
2523 | 0 | unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; |
2524 | | |
2525 | | /* get the converter state from UConverter */ |
2526 | 0 | offset=cnv->toUnicodeStatus; |
2527 | 0 | byteIndex=cnv->toULength; |
2528 | 0 | bytes=cnv->toUBytes; |
2529 | | |
2530 | | /* |
2531 | | * if we are in the SBCS state for a DBCS-only converter, |
2532 | | * then load the DBCS state from the MBCS data |
2533 | | * (dbcsOnlyState==0 if it is not a DBCS-only converter) |
2534 | | */ |
2535 | 0 | if((state=(uint8_t)(cnv->mode))==0) { |
2536 | 0 | state=cnv->sharedData->mbcs.dbcsOnlyState; |
2537 | 0 | } |
2538 | | |
2539 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
2540 | 0 | sourceIndex=byteIndex==0 ? 0 : -1; |
2541 | 0 | nextSourceIndex=0; |
2542 | | |
2543 | | /* conversion loop */ |
2544 | 0 | while(source<sourceLimit) { |
2545 | | /* |
2546 | | * This following test is to see if available input would overflow the output. |
2547 | | * It does not catch output of more than one code unit that |
2548 | | * overflows as a result of a surrogate pair or callback output |
2549 | | * from the last source byte. |
2550 | | * Therefore, those situations also test for overflows and will |
2551 | | * then break the loop, too. |
2552 | | */ |
2553 | 0 | if(target>=targetLimit) { |
2554 | | /* target is full */ |
2555 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2556 | 0 | break; |
2557 | 0 | } |
2558 | | |
2559 | 0 | if(byteIndex==0) { |
2560 | | /* optimized loop for 1/2-byte input and BMP output */ |
2561 | 0 | if(offsets==NULL) { |
2562 | 0 | do { |
2563 | 0 | entry=stateTable[state][*source]; |
2564 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
2565 | 0 | state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); |
2566 | 0 | offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); |
2567 | |
|
2568 | 0 | ++source; |
2569 | 0 | if( source<sourceLimit && |
2570 | 0 | MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) && |
2571 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 && |
2572 | 0 | (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe |
2573 | 0 | ) { |
2574 | 0 | ++source; |
2575 | 0 | *target++=c; |
2576 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
2577 | 0 | offset=0; |
2578 | 0 | } else { |
2579 | | /* set the state and leave the optimized loop */ |
2580 | 0 | bytes[0]=*(source-1); |
2581 | 0 | byteIndex=1; |
2582 | 0 | break; |
2583 | 0 | } |
2584 | 0 | } else { |
2585 | 0 | if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { |
2586 | | /* output BMP code point */ |
2587 | 0 | ++source; |
2588 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2589 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
2590 | 0 | } else { |
2591 | | /* leave the optimized loop */ |
2592 | 0 | break; |
2593 | 0 | } |
2594 | 0 | } |
2595 | 0 | } while(source<sourceLimit && target<targetLimit); |
2596 | 0 | } else /* offsets!=NULL */ { |
2597 | 0 | do { |
2598 | 0 | entry=stateTable[state][*source]; |
2599 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
2600 | 0 | state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); |
2601 | 0 | offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); |
2602 | |
|
2603 | 0 | ++source; |
2604 | 0 | if( source<sourceLimit && |
2605 | 0 | MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) && |
2606 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 && |
2607 | 0 | (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe |
2608 | 0 | ) { |
2609 | 0 | ++source; |
2610 | 0 | *target++=c; |
2611 | 0 | if(offsets!=NULL) { |
2612 | 0 | *offsets++=sourceIndex; |
2613 | 0 | sourceIndex=(nextSourceIndex+=2); |
2614 | 0 | } |
2615 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
2616 | 0 | offset=0; |
2617 | 0 | } else { |
2618 | | /* set the state and leave the optimized loop */ |
2619 | 0 | ++nextSourceIndex; |
2620 | 0 | bytes[0]=*(source-1); |
2621 | 0 | byteIndex=1; |
2622 | 0 | break; |
2623 | 0 | } |
2624 | 0 | } else { |
2625 | 0 | if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { |
2626 | | /* output BMP code point */ |
2627 | 0 | ++source; |
2628 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2629 | 0 | if(offsets!=NULL) { |
2630 | 0 | *offsets++=sourceIndex; |
2631 | 0 | sourceIndex=++nextSourceIndex; |
2632 | 0 | } |
2633 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
2634 | 0 | } else { |
2635 | | /* leave the optimized loop */ |
2636 | 0 | break; |
2637 | 0 | } |
2638 | 0 | } |
2639 | 0 | } while(source<sourceLimit && target<targetLimit); |
2640 | 0 | } |
2641 | | |
2642 | | /* |
2643 | | * these tests and break statements could be put inside the loop |
2644 | | * if C had "break outerLoop" like Java |
2645 | | */ |
2646 | 0 | if(source>=sourceLimit) { |
2647 | 0 | break; |
2648 | 0 | } |
2649 | 0 | if(target>=targetLimit) { |
2650 | | /* target is full */ |
2651 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2652 | 0 | break; |
2653 | 0 | } |
2654 | | |
2655 | 0 | ++nextSourceIndex; |
2656 | 0 | bytes[byteIndex++]=*source++; |
2657 | 0 | } else /* byteIndex>0 */ { |
2658 | 0 | ++nextSourceIndex; |
2659 | 0 | entry=stateTable[state][bytes[byteIndex++]=*source++]; |
2660 | 0 | } |
2661 | | |
2662 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
2663 | 0 | state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); |
2664 | 0 | offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); |
2665 | 0 | continue; |
2666 | 0 | } |
2667 | | |
2668 | | /* save the previous state for proper extension mapping with SI/SO-stateful converters */ |
2669 | 0 | cnv->mode=state; |
2670 | | |
2671 | | /* set the next state early so that we can reuse the entry variable */ |
2672 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
2673 | | |
2674 | | /* |
2675 | | * An if-else-if chain provides more reliable performance for |
2676 | | * the most common cases compared to a switch. |
2677 | | */ |
2678 | 0 | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
2679 | 0 | if(action==MBCS_STATE_VALID_16) { |
2680 | 0 | offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); |
2681 | 0 | c=unicodeCodeUnits[offset]; |
2682 | 0 | if(c<0xfffe) { |
2683 | | /* output BMP code point */ |
2684 | 0 | *target++=c; |
2685 | 0 | if(offsets!=NULL) { |
2686 | 0 | *offsets++=sourceIndex; |
2687 | 0 | } |
2688 | 0 | byteIndex=0; |
2689 | 0 | } else if(c==0xfffe) { |
2690 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { |
2691 | | /* output fallback BMP code point */ |
2692 | 0 | *target++=(UChar)entry; |
2693 | 0 | if(offsets!=NULL) { |
2694 | 0 | *offsets++=sourceIndex; |
2695 | 0 | } |
2696 | 0 | byteIndex=0; |
2697 | 0 | } |
2698 | 0 | } else { |
2699 | | /* callback(illegal) */ |
2700 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2701 | 0 | } |
2702 | 0 | } else if(action==MBCS_STATE_VALID_DIRECT_16) { |
2703 | | /* output BMP code point */ |
2704 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2705 | 0 | if(offsets!=NULL) { |
2706 | 0 | *offsets++=sourceIndex; |
2707 | 0 | } |
2708 | 0 | byteIndex=0; |
2709 | 0 | } else if(action==MBCS_STATE_VALID_16_PAIR) { |
2710 | 0 | offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); |
2711 | 0 | c=unicodeCodeUnits[offset++]; |
2712 | 0 | if(c<0xd800) { |
2713 | | /* output BMP code point below 0xd800 */ |
2714 | 0 | *target++=c; |
2715 | 0 | if(offsets!=NULL) { |
2716 | 0 | *offsets++=sourceIndex; |
2717 | 0 | } |
2718 | 0 | byteIndex=0; |
2719 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { |
2720 | | /* output roundtrip or fallback surrogate pair */ |
2721 | 0 | *target++=(UChar)(c&0xdbff); |
2722 | 0 | if(offsets!=NULL) { |
2723 | 0 | *offsets++=sourceIndex; |
2724 | 0 | } |
2725 | 0 | byteIndex=0; |
2726 | 0 | if(target<targetLimit) { |
2727 | 0 | *target++=unicodeCodeUnits[offset]; |
2728 | 0 | if(offsets!=NULL) { |
2729 | 0 | *offsets++=sourceIndex; |
2730 | 0 | } |
2731 | 0 | } else { |
2732 | | /* target overflow */ |
2733 | 0 | cnv->UCharErrorBuffer[0]=unicodeCodeUnits[offset]; |
2734 | 0 | cnv->UCharErrorBufferLength=1; |
2735 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2736 | |
|
2737 | 0 | offset=0; |
2738 | 0 | break; |
2739 | 0 | } |
2740 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { |
2741 | | /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ |
2742 | 0 | *target++=unicodeCodeUnits[offset]; |
2743 | 0 | if(offsets!=NULL) { |
2744 | 0 | *offsets++=sourceIndex; |
2745 | 0 | } |
2746 | 0 | byteIndex=0; |
2747 | 0 | } else if(c==0xffff) { |
2748 | | /* callback(illegal) */ |
2749 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2750 | 0 | } |
2751 | 0 | } else if(action==MBCS_STATE_VALID_DIRECT_20 || |
2752 | 0 | (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) |
2753 | 0 | ) { |
2754 | 0 | entry=MBCS_ENTRY_FINAL_VALUE(entry); |
2755 | | /* output surrogate pair */ |
2756 | 0 | *target++=(UChar)(0xd800|(UChar)(entry>>10)); |
2757 | 0 | if(offsets!=NULL) { |
2758 | 0 | *offsets++=sourceIndex; |
2759 | 0 | } |
2760 | 0 | byteIndex=0; |
2761 | 0 | c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); |
2762 | 0 | if(target<targetLimit) { |
2763 | 0 | *target++=c; |
2764 | 0 | if(offsets!=NULL) { |
2765 | 0 | *offsets++=sourceIndex; |
2766 | 0 | } |
2767 | 0 | } else { |
2768 | | /* target overflow */ |
2769 | 0 | cnv->UCharErrorBuffer[0]=c; |
2770 | 0 | cnv->UCharErrorBufferLength=1; |
2771 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
2772 | |
|
2773 | 0 | offset=0; |
2774 | 0 | break; |
2775 | 0 | } |
2776 | 0 | } else if(action==MBCS_STATE_CHANGE_ONLY) { |
2777 | | /* |
2778 | | * This serves as a state change without any output. |
2779 | | * It is useful for reading simple stateful encodings, |
2780 | | * for example using just Shift-In/Shift-Out codes. |
2781 | | * The 21 unused bits may later be used for more sophisticated |
2782 | | * state transitions. |
2783 | | */ |
2784 | 0 | if(cnv->sharedData->mbcs.dbcsOnlyState==0) { |
2785 | 0 | byteIndex=0; |
2786 | 0 | } else { |
2787 | | /* SI/SO are illegal for DBCS-only conversion */ |
2788 | 0 | state=(uint8_t)(cnv->mode); /* restore the previous state */ |
2789 | | |
2790 | | /* callback(illegal) */ |
2791 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2792 | 0 | } |
2793 | 0 | } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
2794 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv)) { |
2795 | | /* output BMP code point */ |
2796 | 0 | *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2797 | 0 | if(offsets!=NULL) { |
2798 | 0 | *offsets++=sourceIndex; |
2799 | 0 | } |
2800 | 0 | byteIndex=0; |
2801 | 0 | } |
2802 | 0 | } else if(action==MBCS_STATE_UNASSIGNED) { |
2803 | | /* just fall through */ |
2804 | 0 | } else if(action==MBCS_STATE_ILLEGAL) { |
2805 | | /* callback(illegal) */ |
2806 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2807 | 0 | } else { |
2808 | | /* reserved, must never occur */ |
2809 | 0 | byteIndex=0; |
2810 | 0 | } |
2811 | | |
2812 | | /* end of action codes: prepare for a new character */ |
2813 | 0 | offset=0; |
2814 | |
|
2815 | 0 | if(byteIndex==0) { |
2816 | 0 | sourceIndex=nextSourceIndex; |
2817 | 0 | } else if(U_FAILURE(*pErrorCode)) { |
2818 | | /* callback(illegal) */ |
2819 | 0 | if(byteIndex>1) { |
2820 | | /* |
2821 | | * Ticket 5691: consistent illegal sequences: |
2822 | | * - We include at least the first byte in the illegal sequence. |
2823 | | * - If any of the non-initial bytes could be the start of a character, |
2824 | | * we stop the illegal sequence before the first one of those. |
2825 | | */ |
2826 | 0 | UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0); |
2827 | 0 | int8_t i; |
2828 | 0 | for(i=1; |
2829 | 0 | i<byteIndex && !isSingleOrLead(stateTable, state, isDBCSOnly, bytes[i]); |
2830 | 0 | ++i) {} |
2831 | 0 | if(i<byteIndex) { |
2832 | | /* Back out some bytes. */ |
2833 | 0 | int8_t backOutDistance=byteIndex-i; |
2834 | 0 | int32_t bytesFromThisBuffer=(int32_t)(source-(const uint8_t *)pArgs->source); |
2835 | 0 | byteIndex=i; /* length of reported illegal byte sequence */ |
2836 | 0 | if(backOutDistance<=bytesFromThisBuffer) { |
2837 | 0 | source-=backOutDistance; |
2838 | 0 | } else { |
2839 | | /* Back out bytes from the previous buffer: Need to replay them. */ |
2840 | 0 | cnv->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance); |
2841 | | /* preToULength is negative! */ |
2842 | 0 | uprv_memcpy(cnv->preToU, bytes+i, -cnv->preToULength); |
2843 | 0 | source=(const uint8_t *)pArgs->source; |
2844 | 0 | } |
2845 | 0 | } |
2846 | 0 | } |
2847 | 0 | break; |
2848 | 0 | } else /* unassigned sequences indicated with byteIndex>0 */ { |
2849 | | /* try an extension mapping */ |
2850 | 0 | pArgs->source=(const char *)source; |
2851 | 0 | byteIndex=_extToU(cnv, cnv->sharedData, |
2852 | 0 | byteIndex, &source, sourceLimit, |
2853 | 0 | &target, targetLimit, |
2854 | 0 | &offsets, sourceIndex, |
2855 | 0 | pArgs->flush, |
2856 | 0 | pErrorCode); |
2857 | 0 | sourceIndex=nextSourceIndex+=(int32_t)(source-(const uint8_t *)pArgs->source); |
2858 | |
|
2859 | 0 | if(U_FAILURE(*pErrorCode)) { |
2860 | | /* not mappable or buffer overflow */ |
2861 | 0 | break; |
2862 | 0 | } |
2863 | 0 | } |
2864 | 0 | } |
2865 | | |
2866 | | /* set the converter state back into UConverter */ |
2867 | 0 | cnv->toUnicodeStatus=offset; |
2868 | 0 | cnv->mode=state; |
2869 | 0 | cnv->toULength=byteIndex; |
2870 | | |
2871 | | /* write back the updated pointers */ |
2872 | 0 | pArgs->source=(const char *)source; |
2873 | 0 | pArgs->target=target; |
2874 | 0 | pArgs->offsets=offsets; |
2875 | 0 | } |
2876 | | |
2877 | | /* |
2878 | | * This version of ucnv_MBCSGetNextUChar() is optimized for single-byte, single-state codepages. |
2879 | | * We still need a conversion loop in case we find reserved action codes, which are to be ignored. |
2880 | | */ |
2881 | | static UChar32 |
2882 | | ucnv_MBCSSingleGetNextUChar(UConverterToUnicodeArgs *pArgs, |
2883 | 0 | UErrorCode *pErrorCode) { |
2884 | 0 | UConverter *cnv; |
2885 | 0 | const int32_t (*stateTable)[256]; |
2886 | 0 | const uint8_t *source, *sourceLimit; |
2887 | |
|
2888 | 0 | int32_t entry; |
2889 | 0 | uint8_t action; |
2890 | | |
2891 | | /* set up the local pointers */ |
2892 | 0 | cnv=pArgs->converter; |
2893 | 0 | source=(const uint8_t *)pArgs->source; |
2894 | 0 | sourceLimit=(const uint8_t *)pArgs->sourceLimit; |
2895 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
2896 | 0 | stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; |
2897 | 0 | } else { |
2898 | 0 | stateTable=cnv->sharedData->mbcs.stateTable; |
2899 | 0 | } |
2900 | | |
2901 | | /* conversion loop */ |
2902 | 0 | while(source<sourceLimit) { |
2903 | 0 | entry=stateTable[0][*source++]; |
2904 | | /* MBCS_ENTRY_IS_FINAL(entry) */ |
2905 | | |
2906 | | /* write back the updated pointer early so that we can return directly */ |
2907 | 0 | pArgs->source=(const char *)source; |
2908 | |
|
2909 | 0 | if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { |
2910 | | /* output BMP code point */ |
2911 | 0 | return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2912 | 0 | } |
2913 | | |
2914 | | /* |
2915 | | * An if-else-if chain provides more reliable performance for |
2916 | | * the most common cases compared to a switch. |
2917 | | */ |
2918 | 0 | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
2919 | 0 | if( action==MBCS_STATE_VALID_DIRECT_20 || |
2920 | 0 | (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) |
2921 | 0 | ) { |
2922 | | /* output supplementary code point */ |
2923 | 0 | return (UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); |
2924 | 0 | } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
2925 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv)) { |
2926 | | /* output BMP code point */ |
2927 | 0 | return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
2928 | 0 | } |
2929 | 0 | } else if(action==MBCS_STATE_UNASSIGNED) { |
2930 | | /* just fall through */ |
2931 | 0 | } else if(action==MBCS_STATE_ILLEGAL) { |
2932 | | /* callback(illegal) */ |
2933 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
2934 | 0 | } else { |
2935 | | /* reserved, must never occur */ |
2936 | 0 | continue; |
2937 | 0 | } |
2938 | | |
2939 | 0 | if(U_FAILURE(*pErrorCode)) { |
2940 | | /* callback(illegal) */ |
2941 | 0 | break; |
2942 | 0 | } else /* unassigned sequence */ { |
2943 | | /* defer to the generic implementation */ |
2944 | 0 | pArgs->source=(const char *)source-1; |
2945 | 0 | return UCNV_GET_NEXT_UCHAR_USE_TO_U; |
2946 | 0 | } |
2947 | 0 | } |
2948 | | |
2949 | | /* no output because of empty input or only state changes */ |
2950 | 0 | *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; |
2951 | 0 | return 0xffff; |
2952 | 0 | } |
2953 | | |
2954 | | /* |
2955 | | * Version of _MBCSToUnicodeWithOffsets() optimized for single-character |
2956 | | * conversion without offset handling. |
2957 | | * |
2958 | | * When a character does not have a mapping to Unicode, then we return to the |
2959 | | * generic ucnv_getNextUChar() code for extension/GB 18030 and error/callback |
2960 | | * handling. |
2961 | | * We also defer to the generic code in other complicated cases and have them |
2962 | | * ultimately handled by _MBCSToUnicodeWithOffsets() itself. |
2963 | | * |
2964 | | * All normal mappings and errors are handled here. |
2965 | | */ |
2966 | | static UChar32 U_CALLCONV |
2967 | | ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, |
2968 | 0 | UErrorCode *pErrorCode) { |
2969 | 0 | UConverter *cnv; |
2970 | 0 | const uint8_t *source, *sourceLimit, *lastSource; |
2971 | |
|
2972 | 0 | const int32_t (*stateTable)[256]; |
2973 | 0 | const uint16_t *unicodeCodeUnits; |
2974 | |
|
2975 | 0 | uint32_t offset; |
2976 | 0 | uint8_t state; |
2977 | |
|
2978 | 0 | int32_t entry; |
2979 | 0 | UChar32 c; |
2980 | 0 | uint8_t action; |
2981 | | |
2982 | | /* use optimized function if possible */ |
2983 | 0 | cnv=pArgs->converter; |
2984 | |
|
2985 | 0 | if(cnv->preToULength>0) { |
2986 | | /* use the generic code in ucnv_getNextUChar() to continue with a partial match */ |
2987 | 0 | return UCNV_GET_NEXT_UCHAR_USE_TO_U; |
2988 | 0 | } |
2989 | | |
2990 | 0 | if(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SURROGATES) { |
2991 | | /* |
2992 | | * Using the generic ucnv_getNextUChar() code lets us deal correctly |
2993 | | * with the rare case of a codepage that maps single surrogates |
2994 | | * without adding the complexity to this already complicated function here. |
2995 | | */ |
2996 | 0 | return UCNV_GET_NEXT_UCHAR_USE_TO_U; |
2997 | 0 | } else if(cnv->sharedData->mbcs.countStates==1) { |
2998 | 0 | return ucnv_MBCSSingleGetNextUChar(pArgs, pErrorCode); |
2999 | 0 | } |
3000 | | |
3001 | | /* set up the local pointers */ |
3002 | 0 | source=lastSource=(const uint8_t *)pArgs->source; |
3003 | 0 | sourceLimit=(const uint8_t *)pArgs->sourceLimit; |
3004 | |
|
3005 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
3006 | 0 | stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; |
3007 | 0 | } else { |
3008 | 0 | stateTable=cnv->sharedData->mbcs.stateTable; |
3009 | 0 | } |
3010 | 0 | unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; |
3011 | | |
3012 | | /* get the converter state from UConverter */ |
3013 | 0 | offset=cnv->toUnicodeStatus; |
3014 | | |
3015 | | /* |
3016 | | * if we are in the SBCS state for a DBCS-only converter, |
3017 | | * then load the DBCS state from the MBCS data |
3018 | | * (dbcsOnlyState==0 if it is not a DBCS-only converter) |
3019 | | */ |
3020 | 0 | if((state=(uint8_t)(cnv->mode))==0) { |
3021 | 0 | state=cnv->sharedData->mbcs.dbcsOnlyState; |
3022 | 0 | } |
3023 | | |
3024 | | /* conversion loop */ |
3025 | 0 | c=U_SENTINEL; |
3026 | 0 | while(source<sourceLimit) { |
3027 | 0 | entry=stateTable[state][*source++]; |
3028 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
3029 | 0 | state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); |
3030 | 0 | offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); |
3031 | | |
3032 | | /* optimization for 1/2-byte input and BMP output */ |
3033 | 0 | if( source<sourceLimit && |
3034 | 0 | MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) && |
3035 | 0 | MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 && |
3036 | 0 | (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe |
3037 | 0 | ) { |
3038 | 0 | ++source; |
3039 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
3040 | | /* output BMP code point */ |
3041 | 0 | break; |
3042 | 0 | } |
3043 | 0 | } else { |
3044 | | /* save the previous state for proper extension mapping with SI/SO-stateful converters */ |
3045 | 0 | cnv->mode=state; |
3046 | | |
3047 | | /* set the next state early so that we can reuse the entry variable */ |
3048 | 0 | state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ |
3049 | | |
3050 | | /* |
3051 | | * An if-else-if chain provides more reliable performance for |
3052 | | * the most common cases compared to a switch. |
3053 | | */ |
3054 | 0 | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
3055 | 0 | if(action==MBCS_STATE_VALID_DIRECT_16) { |
3056 | | /* output BMP code point */ |
3057 | 0 | c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
3058 | 0 | break; |
3059 | 0 | } else if(action==MBCS_STATE_VALID_16) { |
3060 | 0 | offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); |
3061 | 0 | c=unicodeCodeUnits[offset]; |
3062 | 0 | if(c<0xfffe) { |
3063 | | /* output BMP code point */ |
3064 | 0 | break; |
3065 | 0 | } else if(c==0xfffe) { |
3066 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv) && (c=ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { |
3067 | 0 | break; |
3068 | 0 | } |
3069 | 0 | } else { |
3070 | | /* callback(illegal) */ |
3071 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3072 | 0 | } |
3073 | 0 | } else if(action==MBCS_STATE_VALID_16_PAIR) { |
3074 | 0 | offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); |
3075 | 0 | c=unicodeCodeUnits[offset++]; |
3076 | 0 | if(c<0xd800) { |
3077 | | /* output BMP code point below 0xd800 */ |
3078 | 0 | break; |
3079 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { |
3080 | | /* output roundtrip or fallback supplementary code point */ |
3081 | 0 | c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00); |
3082 | 0 | break; |
3083 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { |
3084 | | /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ |
3085 | 0 | c=unicodeCodeUnits[offset]; |
3086 | 0 | break; |
3087 | 0 | } else if(c==0xffff) { |
3088 | | /* callback(illegal) */ |
3089 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3090 | 0 | } |
3091 | 0 | } else if(action==MBCS_STATE_VALID_DIRECT_20 || |
3092 | 0 | (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) |
3093 | 0 | ) { |
3094 | | /* output supplementary code point */ |
3095 | 0 | c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); |
3096 | 0 | break; |
3097 | 0 | } else if(action==MBCS_STATE_CHANGE_ONLY) { |
3098 | | /* |
3099 | | * This serves as a state change without any output. |
3100 | | * It is useful for reading simple stateful encodings, |
3101 | | * for example using just Shift-In/Shift-Out codes. |
3102 | | * The 21 unused bits may later be used for more sophisticated |
3103 | | * state transitions. |
3104 | | */ |
3105 | 0 | if(cnv->sharedData->mbcs.dbcsOnlyState!=0) { |
3106 | | /* SI/SO are illegal for DBCS-only conversion */ |
3107 | 0 | state=(uint8_t)(cnv->mode); /* restore the previous state */ |
3108 | | |
3109 | | /* callback(illegal) */ |
3110 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3111 | 0 | } |
3112 | 0 | } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
3113 | 0 | if(UCNV_TO_U_USE_FALLBACK(cnv)) { |
3114 | | /* output BMP code point */ |
3115 | 0 | c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
3116 | 0 | break; |
3117 | 0 | } |
3118 | 0 | } else if(action==MBCS_STATE_UNASSIGNED) { |
3119 | | /* just fall through */ |
3120 | 0 | } else if(action==MBCS_STATE_ILLEGAL) { |
3121 | | /* callback(illegal) */ |
3122 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3123 | 0 | } else { |
3124 | | /* reserved (must never occur), or only state change */ |
3125 | 0 | offset=0; |
3126 | 0 | lastSource=source; |
3127 | 0 | continue; |
3128 | 0 | } |
3129 | | |
3130 | | /* end of action codes: prepare for a new character */ |
3131 | 0 | offset=0; |
3132 | |
|
3133 | 0 | if(U_FAILURE(*pErrorCode)) { |
3134 | | /* callback(illegal) */ |
3135 | 0 | break; |
3136 | 0 | } else /* unassigned sequence */ { |
3137 | | /* defer to the generic implementation */ |
3138 | 0 | cnv->toUnicodeStatus=0; |
3139 | 0 | cnv->mode=state; |
3140 | 0 | pArgs->source=(const char *)lastSource; |
3141 | 0 | return UCNV_GET_NEXT_UCHAR_USE_TO_U; |
3142 | 0 | } |
3143 | 0 | } |
3144 | 0 | } |
3145 | | |
3146 | 0 | if(c<0) { |
3147 | 0 | if(U_SUCCESS(*pErrorCode) && source==sourceLimit && lastSource<source) { |
3148 | | /* incomplete character byte sequence */ |
3149 | 0 | uint8_t *bytes=cnv->toUBytes; |
3150 | 0 | cnv->toULength=(int8_t)(source-lastSource); |
3151 | 0 | do { |
3152 | 0 | *bytes++=*lastSource++; |
3153 | 0 | } while(lastSource<source); |
3154 | 0 | *pErrorCode=U_TRUNCATED_CHAR_FOUND; |
3155 | 0 | } else if(U_FAILURE(*pErrorCode)) { |
3156 | | /* callback(illegal) */ |
3157 | | /* |
3158 | | * Ticket 5691: consistent illegal sequences: |
3159 | | * - We include at least the first byte in the illegal sequence. |
3160 | | * - If any of the non-initial bytes could be the start of a character, |
3161 | | * we stop the illegal sequence before the first one of those. |
3162 | | */ |
3163 | 0 | UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0); |
3164 | 0 | uint8_t *bytes=cnv->toUBytes; |
3165 | 0 | *bytes++=*lastSource++; /* first byte */ |
3166 | 0 | if(lastSource==source) { |
3167 | 0 | cnv->toULength=1; |
3168 | 0 | } else /* lastSource<source: multi-byte character */ { |
3169 | 0 | int8_t i; |
3170 | 0 | for(i=1; |
3171 | 0 | lastSource<source && !isSingleOrLead(stateTable, state, isDBCSOnly, *lastSource); |
3172 | 0 | ++i |
3173 | 0 | ) { |
3174 | 0 | *bytes++=*lastSource++; |
3175 | 0 | } |
3176 | 0 | cnv->toULength=i; |
3177 | 0 | source=lastSource; |
3178 | 0 | } |
3179 | 0 | } else { |
3180 | | /* no output because of empty input or only state changes */ |
3181 | 0 | *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; |
3182 | 0 | } |
3183 | 0 | c=0xffff; |
3184 | 0 | } |
3185 | | |
3186 | | /* set the converter state back into UConverter, ready for a new character */ |
3187 | 0 | cnv->toUnicodeStatus=0; |
3188 | 0 | cnv->mode=state; |
3189 | | |
3190 | | /* write back the updated pointer */ |
3191 | 0 | pArgs->source=(const char *)source; |
3192 | 0 | return c; |
3193 | 0 | } |
3194 | | |
3195 | | #if 0 |
3196 | | /* |
3197 | | * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus |
3198 | | * Removal improves code coverage. |
3199 | | */ |
3200 | | /** |
3201 | | * This version of ucnv_MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. |
3202 | | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
3203 | | * It does not handle conversion extensions (_extToU()). |
3204 | | */ |
3205 | | U_CFUNC UChar32 |
3206 | | ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, |
3207 | | uint8_t b, UBool useFallback) { |
3208 | | int32_t entry; |
3209 | | uint8_t action; |
3210 | | |
3211 | | entry=sharedData->mbcs.stateTable[0][b]; |
3212 | | /* MBCS_ENTRY_IS_FINAL(entry) */ |
3213 | | |
3214 | | if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { |
3215 | | /* output BMP code point */ |
3216 | | return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
3217 | | } |
3218 | | |
3219 | | /* |
3220 | | * An if-else-if chain provides more reliable performance for |
3221 | | * the most common cases compared to a switch. |
3222 | | */ |
3223 | | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
3224 | | if(action==MBCS_STATE_VALID_DIRECT_20) { |
3225 | | /* output supplementary code point */ |
3226 | | return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); |
3227 | | } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
3228 | | if(!TO_U_USE_FALLBACK(useFallback)) { |
3229 | | return 0xfffe; |
3230 | | } |
3231 | | /* output BMP code point */ |
3232 | | return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
3233 | | } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { |
3234 | | if(!TO_U_USE_FALLBACK(useFallback)) { |
3235 | | return 0xfffe; |
3236 | | } |
3237 | | /* output supplementary code point */ |
3238 | | return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); |
3239 | | } else if(action==MBCS_STATE_UNASSIGNED) { |
3240 | | return 0xfffe; |
3241 | | } else if(action==MBCS_STATE_ILLEGAL) { |
3242 | | return 0xffff; |
3243 | | } else { |
3244 | | /* reserved, must never occur */ |
3245 | | return 0xffff; |
3246 | | } |
3247 | | } |
3248 | | #endif |
3249 | | |
3250 | | /* |
3251 | | * This is a simple version of _MBCSGetNextUChar() that is used |
3252 | | * by other converter implementations. |
3253 | | * It only returns an "assigned" result if it consumes the entire input. |
3254 | | * It does not use state from the converter, nor error codes. |
3255 | | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
3256 | | * It handles conversion extensions but not GB 18030. |
3257 | | * |
3258 | | * Return value: |
3259 | | * U+fffe unassigned |
3260 | | * U+ffff illegal |
3261 | | * otherwise the Unicode code point |
3262 | | */ |
3263 | | U_CFUNC UChar32 |
3264 | | ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, |
3265 | | const char *source, int32_t length, |
3266 | 0 | UBool useFallback) { |
3267 | 0 | const int32_t (*stateTable)[256]; |
3268 | 0 | const uint16_t *unicodeCodeUnits; |
3269 | |
|
3270 | 0 | uint32_t offset; |
3271 | 0 | uint8_t state, action; |
3272 | |
|
3273 | 0 | UChar32 c; |
3274 | 0 | int32_t i, entry; |
3275 | |
|
3276 | 0 | if(length<=0) { |
3277 | | /* no input at all: "illegal" */ |
3278 | 0 | return 0xffff; |
3279 | 0 | } |
3280 | | |
3281 | | #if 0 |
3282 | | /* |
3283 | | * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus |
3284 | | * TODO In future releases, verify that this function is never called for SBCS |
3285 | | * conversions, i.e., that sharedData->mbcs.countStates==1 is still true. |
3286 | | * Removal improves code coverage. |
3287 | | */ |
3288 | | /* use optimized function if possible */ |
3289 | | if(sharedData->mbcs.countStates==1) { |
3290 | | if(length==1) { |
3291 | | return ucnv_MBCSSingleSimpleGetNextUChar(sharedData, (uint8_t)*source, useFallback); |
3292 | | } else { |
3293 | | return 0xffff; /* illegal: more than a single byte for an SBCS converter */ |
3294 | | } |
3295 | | } |
3296 | | #endif |
3297 | | |
3298 | | /* set up the local pointers */ |
3299 | 0 | stateTable=sharedData->mbcs.stateTable; |
3300 | 0 | unicodeCodeUnits=sharedData->mbcs.unicodeCodeUnits; |
3301 | | |
3302 | | /* converter state */ |
3303 | 0 | offset=0; |
3304 | 0 | state=sharedData->mbcs.dbcsOnlyState; |
3305 | | |
3306 | | /* conversion loop */ |
3307 | 0 | for(i=0;;) { |
3308 | 0 | entry=stateTable[state][(uint8_t)source[i++]]; |
3309 | 0 | if(MBCS_ENTRY_IS_TRANSITION(entry)) { |
3310 | 0 | state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); |
3311 | 0 | offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); |
3312 | |
|
3313 | 0 | if(i==length) { |
3314 | 0 | return 0xffff; /* truncated character */ |
3315 | 0 | } |
3316 | 0 | } else { |
3317 | | /* |
3318 | | * An if-else-if chain provides more reliable performance for |
3319 | | * the most common cases compared to a switch. |
3320 | | */ |
3321 | 0 | action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); |
3322 | 0 | if(action==MBCS_STATE_VALID_16) { |
3323 | 0 | offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); |
3324 | 0 | c=unicodeCodeUnits[offset]; |
3325 | 0 | if(c!=0xfffe) { |
3326 | | /* done */ |
3327 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv)) { |
3328 | 0 | c=ucnv_MBCSGetFallback(&sharedData->mbcs, offset); |
3329 | | /* else done with 0xfffe */ |
3330 | 0 | } |
3331 | 0 | break; |
3332 | 0 | } else if(action==MBCS_STATE_VALID_DIRECT_16) { |
3333 | | /* output BMP code point */ |
3334 | 0 | c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
3335 | 0 | break; |
3336 | 0 | } else if(action==MBCS_STATE_VALID_16_PAIR) { |
3337 | 0 | offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); |
3338 | 0 | c=unicodeCodeUnits[offset++]; |
3339 | 0 | if(c<0xd800) { |
3340 | | /* output BMP code point below 0xd800 */ |
3341 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { |
3342 | | /* output roundtrip or fallback supplementary code point */ |
3343 | 0 | c=(UChar32)(((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00)); |
3344 | 0 | } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { |
3345 | | /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ |
3346 | 0 | c=unicodeCodeUnits[offset]; |
3347 | 0 | } else if(c==0xffff) { |
3348 | 0 | return 0xffff; |
3349 | 0 | } else { |
3350 | 0 | c=0xfffe; |
3351 | 0 | } |
3352 | 0 | break; |
3353 | 0 | } else if(action==MBCS_STATE_VALID_DIRECT_20) { |
3354 | | /* output supplementary code point */ |
3355 | 0 | c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); |
3356 | 0 | break; |
3357 | 0 | } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { |
3358 | 0 | if(!TO_U_USE_FALLBACK(useFallback)) { |
3359 | 0 | c=0xfffe; |
3360 | 0 | break; |
3361 | 0 | } |
3362 | | /* output BMP code point */ |
3363 | 0 | c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); |
3364 | 0 | break; |
3365 | 0 | } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { |
3366 | 0 | if(!TO_U_USE_FALLBACK(useFallback)) { |
3367 | 0 | c=0xfffe; |
3368 | 0 | break; |
3369 | 0 | } |
3370 | | /* output supplementary code point */ |
3371 | 0 | c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); |
3372 | 0 | break; |
3373 | 0 | } else if(action==MBCS_STATE_UNASSIGNED) { |
3374 | 0 | c=0xfffe; |
3375 | 0 | break; |
3376 | 0 | } |
3377 | | |
3378 | | /* |
3379 | | * forbid MBCS_STATE_CHANGE_ONLY for this function, |
3380 | | * and MBCS_STATE_ILLEGAL and reserved action codes |
3381 | | */ |
3382 | 0 | return 0xffff; |
3383 | 0 | } |
3384 | 0 | } |
3385 | | |
3386 | 0 | if(i!=length) { |
3387 | | /* illegal for this function: not all input consumed */ |
3388 | 0 | return 0xffff; |
3389 | 0 | } |
3390 | | |
3391 | 0 | if(c==0xfffe) { |
3392 | | /* try an extension mapping */ |
3393 | 0 | const int32_t *cx=sharedData->mbcs.extIndexes; |
3394 | 0 | if(cx!=NULL) { |
3395 | 0 | return ucnv_extSimpleMatchToU(cx, source, length, useFallback); |
3396 | 0 | } |
3397 | 0 | } |
3398 | | |
3399 | 0 | return c; |
3400 | 0 | } |
3401 | | |
3402 | | /* MBCS-from-Unicode conversion functions ----------------------------------- */ |
3403 | | |
3404 | | /* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for double-byte codepages. */ |
3405 | | static void |
3406 | | ucnv_MBCSDoubleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, |
3407 | 0 | UErrorCode *pErrorCode) { |
3408 | 0 | UConverter *cnv; |
3409 | 0 | const UChar *source, *sourceLimit; |
3410 | 0 | uint8_t *target; |
3411 | 0 | int32_t targetCapacity; |
3412 | 0 | int32_t *offsets; |
3413 | |
|
3414 | 0 | const uint16_t *table; |
3415 | 0 | const uint16_t *mbcsIndex; |
3416 | 0 | const uint8_t *bytes; |
3417 | |
|
3418 | 0 | UChar32 c; |
3419 | |
|
3420 | 0 | int32_t sourceIndex, nextSourceIndex; |
3421 | |
|
3422 | 0 | uint32_t stage2Entry; |
3423 | 0 | uint32_t asciiRoundtrips; |
3424 | 0 | uint32_t value; |
3425 | 0 | uint8_t unicodeMask; |
3426 | | |
3427 | | /* use optimized function if possible */ |
3428 | 0 | cnv=pArgs->converter; |
3429 | 0 | unicodeMask=cnv->sharedData->mbcs.unicodeMask; |
3430 | | |
3431 | | /* set up the local pointers */ |
3432 | 0 | source=pArgs->source; |
3433 | 0 | sourceLimit=pArgs->sourceLimit; |
3434 | 0 | target=(uint8_t *)pArgs->target; |
3435 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); |
3436 | 0 | offsets=pArgs->offsets; |
3437 | |
|
3438 | 0 | table=cnv->sharedData->mbcs.fromUnicodeTable; |
3439 | 0 | mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; |
3440 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
3441 | 0 | bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; |
3442 | 0 | } else { |
3443 | 0 | bytes=cnv->sharedData->mbcs.fromUnicodeBytes; |
3444 | 0 | } |
3445 | 0 | asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; |
3446 | | |
3447 | | /* get the converter state from UConverter */ |
3448 | 0 | c=cnv->fromUChar32; |
3449 | | |
3450 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
3451 | 0 | sourceIndex= c==0 ? 0 : -1; |
3452 | 0 | nextSourceIndex=0; |
3453 | | |
3454 | | /* conversion loop */ |
3455 | 0 | if(c!=0 && targetCapacity>0) { |
3456 | 0 | goto getTrail; |
3457 | 0 | } |
3458 | | |
3459 | 0 | while(source<sourceLimit) { |
3460 | | /* |
3461 | | * This following test is to see if available input would overflow the output. |
3462 | | * It does not catch output of more than one byte that |
3463 | | * overflows as a result of a multi-byte character or callback output |
3464 | | * from the last source character. |
3465 | | * Therefore, those situations also test for overflows and will |
3466 | | * then break the loop, too. |
3467 | | */ |
3468 | 0 | if(targetCapacity>0) { |
3469 | | /* |
3470 | | * Get a correct Unicode code point: |
3471 | | * a single UChar for a BMP code point or |
3472 | | * a matched surrogate pair for a "supplementary code point". |
3473 | | */ |
3474 | 0 | c=*source++; |
3475 | 0 | ++nextSourceIndex; |
3476 | 0 | if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { |
3477 | 0 | *target++=(uint8_t)c; |
3478 | 0 | if(offsets!=NULL) { |
3479 | 0 | *offsets++=sourceIndex; |
3480 | 0 | sourceIndex=nextSourceIndex; |
3481 | 0 | } |
3482 | 0 | --targetCapacity; |
3483 | 0 | c=0; |
3484 | 0 | continue; |
3485 | 0 | } |
3486 | | /* |
3487 | | * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX |
3488 | | * to avoid dealing with surrogates. |
3489 | | * MBCS_FAST_MAX must be >=0xd7ff. |
3490 | | */ |
3491 | 0 | if(c<=0xd7ff) { |
3492 | 0 | value=DBCS_RESULT_FROM_MOST_BMP(mbcsIndex, (const uint16_t *)bytes, c); |
3493 | | /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ |
3494 | 0 | if(value==0) { |
3495 | 0 | goto unassigned; |
3496 | 0 | } |
3497 | | /* output the value */ |
3498 | 0 | } else { |
3499 | | /* |
3500 | | * This also tests if the codepage maps single surrogates. |
3501 | | * If it does, then surrogates are not paired but mapped separately. |
3502 | | * Note that in this case unmatched surrogates are not detected. |
3503 | | */ |
3504 | 0 | if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { |
3505 | 0 | if(U16_IS_SURROGATE_LEAD(c)) { |
3506 | 0 | getTrail: |
3507 | 0 | if(source<sourceLimit) { |
3508 | | /* test the following code unit */ |
3509 | 0 | UChar trail=*source; |
3510 | 0 | if(U16_IS_TRAIL(trail)) { |
3511 | 0 | ++source; |
3512 | 0 | ++nextSourceIndex; |
3513 | 0 | c=U16_GET_SUPPLEMENTARY(c, trail); |
3514 | 0 | if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { |
3515 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
3516 | | /* callback(unassigned) */ |
3517 | 0 | goto unassigned; |
3518 | 0 | } |
3519 | | /* convert this supplementary code point */ |
3520 | | /* exit this condition tree */ |
3521 | 0 | } else { |
3522 | | /* this is an unmatched lead code unit (1st surrogate) */ |
3523 | | /* callback(illegal) */ |
3524 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3525 | 0 | break; |
3526 | 0 | } |
3527 | 0 | } else { |
3528 | | /* no more input */ |
3529 | 0 | break; |
3530 | 0 | } |
3531 | 0 | } else { |
3532 | | /* this is an unmatched trail code unit (2nd surrogate) */ |
3533 | | /* callback(illegal) */ |
3534 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3535 | 0 | break; |
3536 | 0 | } |
3537 | 0 | } |
3538 | | |
3539 | | /* convert the Unicode code point in c into codepage bytes */ |
3540 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, c); |
3541 | | |
3542 | | /* get the bytes and the length for the output */ |
3543 | | /* MBCS_OUTPUT_2 */ |
3544 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); |
3545 | | |
3546 | | /* is this code point assigned, or do we use fallbacks? */ |
3547 | 0 | if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || |
3548 | 0 | (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) |
3549 | 0 | ) { |
3550 | | /* |
3551 | | * We allow a 0 byte output if the "assigned" bit is set for this entry. |
3552 | | * There is no way with this data structure for fallback output |
3553 | | * to be a zero byte. |
3554 | | */ |
3555 | |
|
3556 | 0 | unassigned: |
3557 | | /* try an extension mapping */ |
3558 | 0 | pArgs->source=source; |
3559 | 0 | c=_extFromU(cnv, cnv->sharedData, |
3560 | 0 | c, &source, sourceLimit, |
3561 | 0 | &target, target+targetCapacity, |
3562 | 0 | &offsets, sourceIndex, |
3563 | 0 | pArgs->flush, |
3564 | 0 | pErrorCode); |
3565 | 0 | nextSourceIndex+=(int32_t)(source-pArgs->source); |
3566 | |
|
3567 | 0 | if(U_FAILURE(*pErrorCode)) { |
3568 | | /* not mappable or buffer overflow */ |
3569 | 0 | break; |
3570 | 0 | } else { |
3571 | | /* a mapping was written to the target, continue */ |
3572 | | |
3573 | | /* recalculate the targetCapacity after an extension mapping */ |
3574 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); |
3575 | | |
3576 | | /* normal end of conversion: prepare for a new character */ |
3577 | 0 | sourceIndex=nextSourceIndex; |
3578 | 0 | continue; |
3579 | 0 | } |
3580 | 0 | } |
3581 | 0 | } |
3582 | | |
3583 | | /* write the output character bytes from value and length */ |
3584 | | /* from the first if in the loop we know that targetCapacity>0 */ |
3585 | 0 | if(value<=0xff) { |
3586 | | /* this is easy because we know that there is enough space */ |
3587 | 0 | *target++=(uint8_t)value; |
3588 | 0 | if(offsets!=NULL) { |
3589 | 0 | *offsets++=sourceIndex; |
3590 | 0 | } |
3591 | 0 | --targetCapacity; |
3592 | 0 | } else /* length==2 */ { |
3593 | 0 | *target++=(uint8_t)(value>>8); |
3594 | 0 | if(2<=targetCapacity) { |
3595 | 0 | *target++=(uint8_t)value; |
3596 | 0 | if(offsets!=NULL) { |
3597 | 0 | *offsets++=sourceIndex; |
3598 | 0 | *offsets++=sourceIndex; |
3599 | 0 | } |
3600 | 0 | targetCapacity-=2; |
3601 | 0 | } else { |
3602 | 0 | if(offsets!=NULL) { |
3603 | 0 | *offsets++=sourceIndex; |
3604 | 0 | } |
3605 | 0 | cnv->charErrorBuffer[0]=(char)value; |
3606 | 0 | cnv->charErrorBufferLength=1; |
3607 | | |
3608 | | /* target overflow */ |
3609 | 0 | targetCapacity=0; |
3610 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
3611 | 0 | c=0; |
3612 | 0 | break; |
3613 | 0 | } |
3614 | 0 | } |
3615 | | |
3616 | | /* normal end of conversion: prepare for a new character */ |
3617 | 0 | c=0; |
3618 | 0 | sourceIndex=nextSourceIndex; |
3619 | 0 | continue; |
3620 | 0 | } else { |
3621 | | /* target is full */ |
3622 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
3623 | 0 | break; |
3624 | 0 | } |
3625 | 0 | } |
3626 | | |
3627 | | /* set the converter state back into UConverter */ |
3628 | 0 | cnv->fromUChar32=c; |
3629 | | |
3630 | | /* write back the updated pointers */ |
3631 | 0 | pArgs->source=source; |
3632 | 0 | pArgs->target=(char *)target; |
3633 | 0 | pArgs->offsets=offsets; |
3634 | 0 | } |
3635 | | |
3636 | | /* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for single-byte codepages. */ |
3637 | | static void |
3638 | | ucnv_MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, |
3639 | 0 | UErrorCode *pErrorCode) { |
3640 | 0 | UConverter *cnv; |
3641 | 0 | const UChar *source, *sourceLimit; |
3642 | 0 | uint8_t *target; |
3643 | 0 | int32_t targetCapacity; |
3644 | 0 | int32_t *offsets; |
3645 | |
|
3646 | 0 | const uint16_t *table; |
3647 | 0 | const uint16_t *results; |
3648 | |
|
3649 | 0 | UChar32 c; |
3650 | |
|
3651 | 0 | int32_t sourceIndex, nextSourceIndex; |
3652 | |
|
3653 | 0 | uint16_t value, minValue; |
3654 | 0 | UBool hasSupplementary; |
3655 | | |
3656 | | /* set up the local pointers */ |
3657 | 0 | cnv=pArgs->converter; |
3658 | 0 | source=pArgs->source; |
3659 | 0 | sourceLimit=pArgs->sourceLimit; |
3660 | 0 | target=(uint8_t *)pArgs->target; |
3661 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); |
3662 | 0 | offsets=pArgs->offsets; |
3663 | |
|
3664 | 0 | table=cnv->sharedData->mbcs.fromUnicodeTable; |
3665 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
3666 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; |
3667 | 0 | } else { |
3668 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; |
3669 | 0 | } |
3670 | |
|
3671 | 0 | if(cnv->useFallback) { |
3672 | | /* use all roundtrip and fallback results */ |
3673 | 0 | minValue=0x800; |
3674 | 0 | } else { |
3675 | | /* use only roundtrips and fallbacks from private-use characters */ |
3676 | 0 | minValue=0xc00; |
3677 | 0 | } |
3678 | 0 | hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); |
3679 | | |
3680 | | /* get the converter state from UConverter */ |
3681 | 0 | c=cnv->fromUChar32; |
3682 | | |
3683 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
3684 | 0 | sourceIndex= c==0 ? 0 : -1; |
3685 | 0 | nextSourceIndex=0; |
3686 | | |
3687 | | /* conversion loop */ |
3688 | 0 | if(c!=0 && targetCapacity>0) { |
3689 | 0 | goto getTrail; |
3690 | 0 | } |
3691 | | |
3692 | 0 | while(source<sourceLimit) { |
3693 | | /* |
3694 | | * This following test is to see if available input would overflow the output. |
3695 | | * It does not catch output of more than one byte that |
3696 | | * overflows as a result of a multi-byte character or callback output |
3697 | | * from the last source character. |
3698 | | * Therefore, those situations also test for overflows and will |
3699 | | * then break the loop, too. |
3700 | | */ |
3701 | 0 | if(targetCapacity>0) { |
3702 | | /* |
3703 | | * Get a correct Unicode code point: |
3704 | | * a single UChar for a BMP code point or |
3705 | | * a matched surrogate pair for a "supplementary code point". |
3706 | | */ |
3707 | 0 | c=*source++; |
3708 | 0 | ++nextSourceIndex; |
3709 | 0 | if(U16_IS_SURROGATE(c)) { |
3710 | 0 | if(U16_IS_SURROGATE_LEAD(c)) { |
3711 | 0 | getTrail: |
3712 | 0 | if(source<sourceLimit) { |
3713 | | /* test the following code unit */ |
3714 | 0 | UChar trail=*source; |
3715 | 0 | if(U16_IS_TRAIL(trail)) { |
3716 | 0 | ++source; |
3717 | 0 | ++nextSourceIndex; |
3718 | 0 | c=U16_GET_SUPPLEMENTARY(c, trail); |
3719 | 0 | if(!hasSupplementary) { |
3720 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
3721 | | /* callback(unassigned) */ |
3722 | 0 | goto unassigned; |
3723 | 0 | } |
3724 | | /* convert this supplementary code point */ |
3725 | | /* exit this condition tree */ |
3726 | 0 | } else { |
3727 | | /* this is an unmatched lead code unit (1st surrogate) */ |
3728 | | /* callback(illegal) */ |
3729 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3730 | 0 | break; |
3731 | 0 | } |
3732 | 0 | } else { |
3733 | | /* no more input */ |
3734 | 0 | break; |
3735 | 0 | } |
3736 | 0 | } else { |
3737 | | /* this is an unmatched trail code unit (2nd surrogate) */ |
3738 | | /* callback(illegal) */ |
3739 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3740 | 0 | break; |
3741 | 0 | } |
3742 | 0 | } |
3743 | | |
3744 | | /* convert the Unicode code point in c into codepage bytes */ |
3745 | 0 | value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
3746 | | |
3747 | | /* is this code point assigned, or do we use fallbacks? */ |
3748 | 0 | if(value>=minValue) { |
3749 | | /* assigned, write the output character bytes from value and length */ |
3750 | | /* length==1 */ |
3751 | | /* this is easy because we know that there is enough space */ |
3752 | 0 | *target++=(uint8_t)value; |
3753 | 0 | if(offsets!=NULL) { |
3754 | 0 | *offsets++=sourceIndex; |
3755 | 0 | } |
3756 | 0 | --targetCapacity; |
3757 | | |
3758 | | /* normal end of conversion: prepare for a new character */ |
3759 | 0 | c=0; |
3760 | 0 | sourceIndex=nextSourceIndex; |
3761 | 0 | } else { /* unassigned */ |
3762 | 0 | unassigned: |
3763 | | /* try an extension mapping */ |
3764 | 0 | pArgs->source=source; |
3765 | 0 | c=_extFromU(cnv, cnv->sharedData, |
3766 | 0 | c, &source, sourceLimit, |
3767 | 0 | &target, target+targetCapacity, |
3768 | 0 | &offsets, sourceIndex, |
3769 | 0 | pArgs->flush, |
3770 | 0 | pErrorCode); |
3771 | 0 | nextSourceIndex+=(int32_t)(source-pArgs->source); |
3772 | |
|
3773 | 0 | if(U_FAILURE(*pErrorCode)) { |
3774 | | /* not mappable or buffer overflow */ |
3775 | 0 | break; |
3776 | 0 | } else { |
3777 | | /* a mapping was written to the target, continue */ |
3778 | | |
3779 | | /* recalculate the targetCapacity after an extension mapping */ |
3780 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); |
3781 | | |
3782 | | /* normal end of conversion: prepare for a new character */ |
3783 | 0 | sourceIndex=nextSourceIndex; |
3784 | 0 | } |
3785 | 0 | } |
3786 | 0 | } else { |
3787 | | /* target is full */ |
3788 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
3789 | 0 | break; |
3790 | 0 | } |
3791 | 0 | } |
3792 | | |
3793 | | /* set the converter state back into UConverter */ |
3794 | 0 | cnv->fromUChar32=c; |
3795 | | |
3796 | | /* write back the updated pointers */ |
3797 | 0 | pArgs->source=source; |
3798 | 0 | pArgs->target=(char *)target; |
3799 | 0 | pArgs->offsets=offsets; |
3800 | 0 | } |
3801 | | |
3802 | | /* |
3803 | | * This version of ucnv_MBCSFromUnicode() is optimized for single-byte codepages |
3804 | | * that map only to and from the BMP. |
3805 | | * In addition to single-byte/state optimizations, the offset calculations |
3806 | | * become much easier. |
3807 | | * It would be possible to use the sbcsIndex for UTF-8-friendly tables, |
3808 | | * but measurements have shown that this diminishes performance |
3809 | | * in more cases than it improves it. |
3810 | | * See SVN revision 21013 (2007-feb-06) for the last version with #if switches |
3811 | | * for various MBCS and SBCS optimizations. |
3812 | | */ |
3813 | | static void |
3814 | | ucnv_MBCSSingleFromBMPWithOffsets(UConverterFromUnicodeArgs *pArgs, |
3815 | 0 | UErrorCode *pErrorCode) { |
3816 | 0 | UConverter *cnv; |
3817 | 0 | const UChar *source, *sourceLimit, *lastSource; |
3818 | 0 | uint8_t *target; |
3819 | 0 | int32_t targetCapacity, length; |
3820 | 0 | int32_t *offsets; |
3821 | |
|
3822 | 0 | const uint16_t *table; |
3823 | 0 | const uint16_t *results; |
3824 | |
|
3825 | 0 | UChar32 c; |
3826 | |
|
3827 | 0 | int32_t sourceIndex; |
3828 | |
|
3829 | 0 | uint32_t asciiRoundtrips; |
3830 | 0 | uint16_t value, minValue; |
3831 | | |
3832 | | /* set up the local pointers */ |
3833 | 0 | cnv=pArgs->converter; |
3834 | 0 | source=pArgs->source; |
3835 | 0 | sourceLimit=pArgs->sourceLimit; |
3836 | 0 | target=(uint8_t *)pArgs->target; |
3837 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); |
3838 | 0 | offsets=pArgs->offsets; |
3839 | |
|
3840 | 0 | table=cnv->sharedData->mbcs.fromUnicodeTable; |
3841 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
3842 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; |
3843 | 0 | } else { |
3844 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; |
3845 | 0 | } |
3846 | 0 | asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; |
3847 | |
|
3848 | 0 | if(cnv->useFallback) { |
3849 | | /* use all roundtrip and fallback results */ |
3850 | 0 | minValue=0x800; |
3851 | 0 | } else { |
3852 | | /* use only roundtrips and fallbacks from private-use characters */ |
3853 | 0 | minValue=0xc00; |
3854 | 0 | } |
3855 | | |
3856 | | /* get the converter state from UConverter */ |
3857 | 0 | c=cnv->fromUChar32; |
3858 | | |
3859 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
3860 | 0 | sourceIndex= c==0 ? 0 : -1; |
3861 | 0 | lastSource=source; |
3862 | | |
3863 | | /* |
3864 | | * since the conversion here is 1:1 UChar:uint8_t, we need only one counter |
3865 | | * for the minimum of the sourceLength and targetCapacity |
3866 | | */ |
3867 | 0 | length=(int32_t)(sourceLimit-source); |
3868 | 0 | if(length<targetCapacity) { |
3869 | 0 | targetCapacity=length; |
3870 | 0 | } |
3871 | | |
3872 | | /* conversion loop */ |
3873 | 0 | if(c!=0 && targetCapacity>0) { |
3874 | 0 | goto getTrail; |
3875 | 0 | } |
3876 | | |
3877 | | #if MBCS_UNROLL_SINGLE_FROM_BMP |
3878 | | /* unrolling makes it slower on Pentium III/Windows 2000?! */ |
3879 | | /* unroll the loop with the most common case */ |
3880 | | unrolled: |
3881 | | if(targetCapacity>=4) { |
3882 | | int32_t count, loops; |
3883 | | uint16_t andedValues; |
3884 | | |
3885 | | loops=count=targetCapacity>>2; |
3886 | | do { |
3887 | | c=*source++; |
3888 | | andedValues=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
3889 | | *target++=(uint8_t)value; |
3890 | | c=*source++; |
3891 | | andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
3892 | | *target++=(uint8_t)value; |
3893 | | c=*source++; |
3894 | | andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
3895 | | *target++=(uint8_t)value; |
3896 | | c=*source++; |
3897 | | andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
3898 | | *target++=(uint8_t)value; |
3899 | | |
3900 | | /* were all 4 entries really valid? */ |
3901 | | if(andedValues<minValue) { |
3902 | | /* no, return to the first of these 4 */ |
3903 | | source-=4; |
3904 | | target-=4; |
3905 | | break; |
3906 | | } |
3907 | | } while(--count>0); |
3908 | | count=loops-count; |
3909 | | targetCapacity-=4*count; |
3910 | | |
3911 | | if(offsets!=NULL) { |
3912 | | lastSource+=4*count; |
3913 | | while(count>0) { |
3914 | | *offsets++=sourceIndex++; |
3915 | | *offsets++=sourceIndex++; |
3916 | | *offsets++=sourceIndex++; |
3917 | | *offsets++=sourceIndex++; |
3918 | | --count; |
3919 | | } |
3920 | | } |
3921 | | |
3922 | | c=0; |
3923 | | } |
3924 | | #endif |
3925 | | |
3926 | 0 | while(targetCapacity>0) { |
3927 | | /* |
3928 | | * Get a correct Unicode code point: |
3929 | | * a single UChar for a BMP code point or |
3930 | | * a matched surrogate pair for a "supplementary code point". |
3931 | | */ |
3932 | 0 | c=*source++; |
3933 | | /* |
3934 | | * Do not immediately check for single surrogates: |
3935 | | * Assume that they are unassigned and check for them in that case. |
3936 | | * This speeds up the conversion of assigned characters. |
3937 | | */ |
3938 | | /* convert the Unicode code point in c into codepage bytes */ |
3939 | 0 | if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { |
3940 | 0 | *target++=(uint8_t)c; |
3941 | 0 | --targetCapacity; |
3942 | 0 | c=0; |
3943 | 0 | continue; |
3944 | 0 | } |
3945 | 0 | value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
3946 | | /* is this code point assigned, or do we use fallbacks? */ |
3947 | 0 | if(value>=minValue) { |
3948 | | /* assigned, write the output character bytes from value and length */ |
3949 | | /* length==1 */ |
3950 | | /* this is easy because we know that there is enough space */ |
3951 | 0 | *target++=(uint8_t)value; |
3952 | 0 | --targetCapacity; |
3953 | | |
3954 | | /* normal end of conversion: prepare for a new character */ |
3955 | 0 | c=0; |
3956 | 0 | continue; |
3957 | 0 | } else if(!U16_IS_SURROGATE(c)) { |
3958 | | /* normal, unassigned BMP character */ |
3959 | 0 | } else if(U16_IS_SURROGATE_LEAD(c)) { |
3960 | 0 | getTrail: |
3961 | 0 | if(source<sourceLimit) { |
3962 | | /* test the following code unit */ |
3963 | 0 | UChar trail=*source; |
3964 | 0 | if(U16_IS_TRAIL(trail)) { |
3965 | 0 | ++source; |
3966 | 0 | c=U16_GET_SUPPLEMENTARY(c, trail); |
3967 | | /* this codepage does not map supplementary code points */ |
3968 | | /* callback(unassigned) */ |
3969 | 0 | } else { |
3970 | | /* this is an unmatched lead code unit (1st surrogate) */ |
3971 | | /* callback(illegal) */ |
3972 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3973 | 0 | break; |
3974 | 0 | } |
3975 | 0 | } else { |
3976 | | /* no more input */ |
3977 | 0 | if (pArgs->flush) { |
3978 | 0 | *pErrorCode=U_TRUNCATED_CHAR_FOUND; |
3979 | 0 | } |
3980 | 0 | break; |
3981 | 0 | } |
3982 | 0 | } else { |
3983 | | /* this is an unmatched trail code unit (2nd surrogate) */ |
3984 | | /* callback(illegal) */ |
3985 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
3986 | 0 | break; |
3987 | 0 | } |
3988 | | |
3989 | | /* c does not have a mapping */ |
3990 | | |
3991 | | /* get the number of code units for c to correctly advance sourceIndex */ |
3992 | 0 | length=U16_LENGTH(c); |
3993 | | |
3994 | | /* set offsets since the start or the last extension */ |
3995 | 0 | if(offsets!=NULL) { |
3996 | 0 | int32_t count=(int32_t)(source-lastSource); |
3997 | | |
3998 | | /* do not set the offset for this character */ |
3999 | 0 | count-=length; |
4000 | |
|
4001 | 0 | while(count>0) { |
4002 | 0 | *offsets++=sourceIndex++; |
4003 | 0 | --count; |
4004 | 0 | } |
4005 | | /* offsets and sourceIndex are now set for the current character */ |
4006 | 0 | } |
4007 | | |
4008 | | /* try an extension mapping */ |
4009 | 0 | lastSource=source; |
4010 | 0 | c=_extFromU(cnv, cnv->sharedData, |
4011 | 0 | c, &source, sourceLimit, |
4012 | 0 | &target, (const uint8_t *)(pArgs->targetLimit), |
4013 | 0 | &offsets, sourceIndex, |
4014 | 0 | pArgs->flush, |
4015 | 0 | pErrorCode); |
4016 | 0 | sourceIndex+=length+(int32_t)(source-lastSource); |
4017 | 0 | lastSource=source; |
4018 | |
|
4019 | 0 | if(U_FAILURE(*pErrorCode)) { |
4020 | | /* not mappable or buffer overflow */ |
4021 | 0 | break; |
4022 | 0 | } else { |
4023 | | /* a mapping was written to the target, continue */ |
4024 | | |
4025 | | /* recalculate the targetCapacity after an extension mapping */ |
4026 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); |
4027 | 0 | length=(int32_t)(sourceLimit-source); |
4028 | 0 | if(length<targetCapacity) { |
4029 | 0 | targetCapacity=length; |
4030 | 0 | } |
4031 | 0 | } |
4032 | |
|
4033 | | #if MBCS_UNROLL_SINGLE_FROM_BMP |
4034 | | /* unrolling makes it slower on Pentium III/Windows 2000?! */ |
4035 | | goto unrolled; |
4036 | | #endif |
4037 | 0 | } |
4038 | | |
4039 | 0 | if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=(uint8_t *)pArgs->targetLimit) { |
4040 | | /* target is full */ |
4041 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
4042 | 0 | } |
4043 | | |
4044 | | /* set offsets since the start or the last callback */ |
4045 | 0 | if(offsets!=NULL) { |
4046 | 0 | size_t count=source-lastSource; |
4047 | 0 | if (count > 0 && *pErrorCode == U_TRUNCATED_CHAR_FOUND) { |
4048 | | /* |
4049 | | Caller gave us a partial supplementary character, |
4050 | | which this function couldn't convert in any case. |
4051 | | The callback will handle the offset. |
4052 | | */ |
4053 | 0 | count--; |
4054 | 0 | } |
4055 | 0 | while(count>0) { |
4056 | 0 | *offsets++=sourceIndex++; |
4057 | 0 | --count; |
4058 | 0 | } |
4059 | 0 | } |
4060 | | |
4061 | | /* set the converter state back into UConverter */ |
4062 | 0 | cnv->fromUChar32=c; |
4063 | | |
4064 | | /* write back the updated pointers */ |
4065 | 0 | pArgs->source=source; |
4066 | 0 | pArgs->target=(char *)target; |
4067 | 0 | pArgs->offsets=offsets; |
4068 | 0 | } |
4069 | | |
4070 | | U_CFUNC void |
4071 | | ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, |
4072 | 0 | UErrorCode *pErrorCode) { |
4073 | 0 | UConverter *cnv; |
4074 | 0 | const UChar *source, *sourceLimit; |
4075 | 0 | uint8_t *target; |
4076 | 0 | int32_t targetCapacity; |
4077 | 0 | int32_t *offsets; |
4078 | |
|
4079 | 0 | const uint16_t *table; |
4080 | 0 | const uint16_t *mbcsIndex; |
4081 | 0 | const uint8_t *p, *bytes; |
4082 | 0 | uint8_t outputType; |
4083 | |
|
4084 | 0 | UChar32 c; |
4085 | |
|
4086 | 0 | int32_t prevSourceIndex, sourceIndex, nextSourceIndex; |
4087 | |
|
4088 | 0 | uint32_t stage2Entry; |
4089 | 0 | uint32_t asciiRoundtrips; |
4090 | 0 | uint32_t value; |
4091 | | /* Shift-In and Shift-Out byte sequences differ by encoding scheme. */ |
4092 | 0 | uint8_t siBytes[2] = {0, 0}; |
4093 | 0 | uint8_t soBytes[2] = {0, 0}; |
4094 | 0 | uint8_t siLength, soLength; |
4095 | 0 | int32_t length = 0, prevLength; |
4096 | 0 | uint8_t unicodeMask; |
4097 | |
|
4098 | 0 | cnv=pArgs->converter; |
4099 | |
|
4100 | 0 | if(cnv->preFromUFirstCP>=0) { |
4101 | | /* |
4102 | | * pass sourceIndex=-1 because we continue from an earlier buffer |
4103 | | * in the future, this may change with continuous offsets |
4104 | | */ |
4105 | 0 | ucnv_extContinueMatchFromU(cnv, pArgs, -1, pErrorCode); |
4106 | |
|
4107 | 0 | if(U_FAILURE(*pErrorCode) || cnv->preFromULength<0) { |
4108 | 0 | return; |
4109 | 0 | } |
4110 | 0 | } |
4111 | | |
4112 | | /* use optimized function if possible */ |
4113 | 0 | outputType=cnv->sharedData->mbcs.outputType; |
4114 | 0 | unicodeMask=cnv->sharedData->mbcs.unicodeMask; |
4115 | 0 | if(outputType==MBCS_OUTPUT_1 && !(unicodeMask&UCNV_HAS_SURROGATES)) { |
4116 | 0 | if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { |
4117 | 0 | ucnv_MBCSSingleFromBMPWithOffsets(pArgs, pErrorCode); |
4118 | 0 | } else { |
4119 | 0 | ucnv_MBCSSingleFromUnicodeWithOffsets(pArgs, pErrorCode); |
4120 | 0 | } |
4121 | 0 | return; |
4122 | 0 | } else if(outputType==MBCS_OUTPUT_2 && cnv->sharedData->mbcs.utf8Friendly) { |
4123 | 0 | ucnv_MBCSDoubleFromUnicodeWithOffsets(pArgs, pErrorCode); |
4124 | 0 | return; |
4125 | 0 | } |
4126 | | |
4127 | | /* set up the local pointers */ |
4128 | 0 | source=pArgs->source; |
4129 | 0 | sourceLimit=pArgs->sourceLimit; |
4130 | 0 | target=(uint8_t *)pArgs->target; |
4131 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); |
4132 | 0 | offsets=pArgs->offsets; |
4133 | |
|
4134 | 0 | table=cnv->sharedData->mbcs.fromUnicodeTable; |
4135 | 0 | if(cnv->sharedData->mbcs.utf8Friendly) { |
4136 | 0 | mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; |
4137 | 0 | } else { |
4138 | 0 | mbcsIndex=NULL; |
4139 | 0 | } |
4140 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
4141 | 0 | bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; |
4142 | 0 | } else { |
4143 | 0 | bytes=cnv->sharedData->mbcs.fromUnicodeBytes; |
4144 | 0 | } |
4145 | 0 | asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; |
4146 | | |
4147 | | /* get the converter state from UConverter */ |
4148 | 0 | c=cnv->fromUChar32; |
4149 | |
|
4150 | 0 | if(outputType==MBCS_OUTPUT_2_SISO) { |
4151 | 0 | prevLength=cnv->fromUnicodeStatus; |
4152 | 0 | if(prevLength==0) { |
4153 | | /* set the real value */ |
4154 | 0 | prevLength=1; |
4155 | 0 | } |
4156 | 0 | } else { |
4157 | | /* prevent fromUnicodeStatus from being set to something non-0 */ |
4158 | 0 | prevLength=0; |
4159 | 0 | } |
4160 | | |
4161 | | /* sourceIndex=-1 if the current character began in the previous buffer */ |
4162 | 0 | prevSourceIndex=-1; |
4163 | 0 | sourceIndex= c==0 ? 0 : -1; |
4164 | 0 | nextSourceIndex=0; |
4165 | | |
4166 | | /* Get the SI/SO character for the converter */ |
4167 | 0 | siLength = static_cast<uint8_t>(getSISOBytes(SI, cnv->options, siBytes)); |
4168 | 0 | soLength = static_cast<uint8_t>(getSISOBytes(SO, cnv->options, soBytes)); |
4169 | | |
4170 | | /* conversion loop */ |
4171 | | /* |
4172 | | * This is another piece of ugly code: |
4173 | | * A goto into the loop if the converter state contains a first surrogate |
4174 | | * from the previous function call. |
4175 | | * It saves me to check in each loop iteration a check of if(c==0) |
4176 | | * and duplicating the trail-surrogate-handling code in the else |
4177 | | * branch of that check. |
4178 | | * I could not find any other way to get around this other than |
4179 | | * using a function call for the conversion and callback, which would |
4180 | | * be even more inefficient. |
4181 | | * |
4182 | | * Markus Scherer 2000-jul-19 |
4183 | | */ |
4184 | 0 | if(c!=0 && targetCapacity>0) { |
4185 | 0 | goto getTrail; |
4186 | 0 | } |
4187 | | |
4188 | 0 | while(source<sourceLimit) { |
4189 | | /* |
4190 | | * This following test is to see if available input would overflow the output. |
4191 | | * It does not catch output of more than one byte that |
4192 | | * overflows as a result of a multi-byte character or callback output |
4193 | | * from the last source character. |
4194 | | * Therefore, those situations also test for overflows and will |
4195 | | * then break the loop, too. |
4196 | | */ |
4197 | 0 | if(targetCapacity>0) { |
4198 | | /* |
4199 | | * Get a correct Unicode code point: |
4200 | | * a single UChar for a BMP code point or |
4201 | | * a matched surrogate pair for a "supplementary code point". |
4202 | | */ |
4203 | 0 | c=*source++; |
4204 | 0 | ++nextSourceIndex; |
4205 | 0 | if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { |
4206 | 0 | *target++=(uint8_t)c; |
4207 | 0 | if(offsets!=NULL) { |
4208 | 0 | *offsets++=sourceIndex; |
4209 | 0 | prevSourceIndex=sourceIndex; |
4210 | 0 | sourceIndex=nextSourceIndex; |
4211 | 0 | } |
4212 | 0 | --targetCapacity; |
4213 | 0 | c=0; |
4214 | 0 | continue; |
4215 | 0 | } |
4216 | | /* |
4217 | | * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX |
4218 | | * to avoid dealing with surrogates. |
4219 | | * MBCS_FAST_MAX must be >=0xd7ff. |
4220 | | */ |
4221 | 0 | if(c<=0xd7ff && mbcsIndex!=NULL) { |
4222 | 0 | value=mbcsIndex[c>>6]; |
4223 | | |
4224 | | /* get the bytes and the length for the output (copied from below and adapted for utf8Friendly data) */ |
4225 | | /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ |
4226 | 0 | switch(outputType) { |
4227 | 0 | case MBCS_OUTPUT_2: |
4228 | 0 | value=((const uint16_t *)bytes)[value +(c&0x3f)]; |
4229 | 0 | if(value<=0xff) { |
4230 | 0 | if(value==0) { |
4231 | 0 | goto unassigned; |
4232 | 0 | } else { |
4233 | 0 | length=1; |
4234 | 0 | } |
4235 | 0 | } else { |
4236 | 0 | length=2; |
4237 | 0 | } |
4238 | 0 | break; |
4239 | 0 | case MBCS_OUTPUT_2_SISO: |
4240 | | /* 1/2-byte stateful with Shift-In/Shift-Out */ |
4241 | | /* |
4242 | | * Save the old state in the converter object |
4243 | | * right here, then change the local prevLength state variable if necessary. |
4244 | | * Then, if this character turns out to be unassigned or a fallback that |
4245 | | * is not taken, the callback code must not save the new state in the converter |
4246 | | * because the new state is for a character that is not output. |
4247 | | * However, the callback must still restore the state from the converter |
4248 | | * in case the callback function changed it for its output. |
4249 | | */ |
4250 | 0 | cnv->fromUnicodeStatus=prevLength; /* save the old state */ |
4251 | 0 | value=((const uint16_t *)bytes)[value +(c&0x3f)]; |
4252 | 0 | if(value<=0xff) { |
4253 | 0 | if(value==0) { |
4254 | 0 | goto unassigned; |
4255 | 0 | } else if(prevLength<=1) { |
4256 | 0 | length=1; |
4257 | 0 | } else { |
4258 | | /* change from double-byte mode to single-byte */ |
4259 | 0 | if (siLength == 1) { |
4260 | 0 | value|=(uint32_t)siBytes[0]<<8; |
4261 | 0 | length = 2; |
4262 | 0 | } else if (siLength == 2) { |
4263 | 0 | value|=(uint32_t)siBytes[1]<<8; |
4264 | 0 | value|=(uint32_t)siBytes[0]<<16; |
4265 | 0 | length = 3; |
4266 | 0 | } |
4267 | 0 | prevLength=1; |
4268 | 0 | } |
4269 | 0 | } else { |
4270 | 0 | if(prevLength==2) { |
4271 | 0 | length=2; |
4272 | 0 | } else { |
4273 | | /* change from single-byte mode to double-byte */ |
4274 | 0 | if (soLength == 1) { |
4275 | 0 | value|=(uint32_t)soBytes[0]<<16; |
4276 | 0 | length = 3; |
4277 | 0 | } else if (soLength == 2) { |
4278 | 0 | value|=(uint32_t)soBytes[1]<<16; |
4279 | 0 | value|=(uint32_t)soBytes[0]<<24; |
4280 | 0 | length = 4; |
4281 | 0 | } |
4282 | 0 | prevLength=2; |
4283 | 0 | } |
4284 | 0 | } |
4285 | 0 | break; |
4286 | 0 | case MBCS_OUTPUT_DBCS_ONLY: |
4287 | | /* table with single-byte results, but only DBCS mappings used */ |
4288 | 0 | value=((const uint16_t *)bytes)[value +(c&0x3f)]; |
4289 | 0 | if(value<=0xff) { |
4290 | | /* no mapping or SBCS result, not taken for DBCS-only */ |
4291 | 0 | goto unassigned; |
4292 | 0 | } else { |
4293 | 0 | length=2; |
4294 | 0 | } |
4295 | 0 | break; |
4296 | 0 | case MBCS_OUTPUT_3: |
4297 | 0 | p=bytes+(value+(c&0x3f))*3; |
4298 | 0 | value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; |
4299 | 0 | if(value<=0xff) { |
4300 | 0 | if(value==0) { |
4301 | 0 | goto unassigned; |
4302 | 0 | } else { |
4303 | 0 | length=1; |
4304 | 0 | } |
4305 | 0 | } else if(value<=0xffff) { |
4306 | 0 | length=2; |
4307 | 0 | } else { |
4308 | 0 | length=3; |
4309 | 0 | } |
4310 | 0 | break; |
4311 | 0 | case MBCS_OUTPUT_4: |
4312 | 0 | value=((const uint32_t *)bytes)[value +(c&0x3f)]; |
4313 | 0 | if(value<=0xff) { |
4314 | 0 | if(value==0) { |
4315 | 0 | goto unassigned; |
4316 | 0 | } else { |
4317 | 0 | length=1; |
4318 | 0 | } |
4319 | 0 | } else if(value<=0xffff) { |
4320 | 0 | length=2; |
4321 | 0 | } else if(value<=0xffffff) { |
4322 | 0 | length=3; |
4323 | 0 | } else { |
4324 | 0 | length=4; |
4325 | 0 | } |
4326 | 0 | break; |
4327 | 0 | case MBCS_OUTPUT_3_EUC: |
4328 | 0 | value=((const uint16_t *)bytes)[value +(c&0x3f)]; |
4329 | | /* EUC 16-bit fixed-length representation */ |
4330 | 0 | if(value<=0xff) { |
4331 | 0 | if(value==0) { |
4332 | 0 | goto unassigned; |
4333 | 0 | } else { |
4334 | 0 | length=1; |
4335 | 0 | } |
4336 | 0 | } else if((value&0x8000)==0) { |
4337 | 0 | value|=0x8e8000; |
4338 | 0 | length=3; |
4339 | 0 | } else if((value&0x80)==0) { |
4340 | 0 | value|=0x8f0080; |
4341 | 0 | length=3; |
4342 | 0 | } else { |
4343 | 0 | length=2; |
4344 | 0 | } |
4345 | 0 | break; |
4346 | 0 | case MBCS_OUTPUT_4_EUC: |
4347 | 0 | p=bytes+(value+(c&0x3f))*3; |
4348 | 0 | value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; |
4349 | | /* EUC 16-bit fixed-length representation applied to the first two bytes */ |
4350 | 0 | if(value<=0xff) { |
4351 | 0 | if(value==0) { |
4352 | 0 | goto unassigned; |
4353 | 0 | } else { |
4354 | 0 | length=1; |
4355 | 0 | } |
4356 | 0 | } else if(value<=0xffff) { |
4357 | 0 | length=2; |
4358 | 0 | } else if((value&0x800000)==0) { |
4359 | 0 | value|=0x8e800000; |
4360 | 0 | length=4; |
4361 | 0 | } else if((value&0x8000)==0) { |
4362 | 0 | value|=0x8f008000; |
4363 | 0 | length=4; |
4364 | 0 | } else { |
4365 | 0 | length=3; |
4366 | 0 | } |
4367 | 0 | break; |
4368 | 0 | default: |
4369 | | /* must not occur */ |
4370 | | /* |
4371 | | * To avoid compiler warnings that value & length may be |
4372 | | * used without having been initialized, we set them here. |
4373 | | * In reality, this is unreachable code. |
4374 | | * Not having a default branch also causes warnings with |
4375 | | * some compilers. |
4376 | | */ |
4377 | 0 | value=0; |
4378 | 0 | length=0; |
4379 | 0 | break; |
4380 | 0 | } |
4381 | | /* output the value */ |
4382 | 0 | } else { |
4383 | | /* |
4384 | | * This also tests if the codepage maps single surrogates. |
4385 | | * If it does, then surrogates are not paired but mapped separately. |
4386 | | * Note that in this case unmatched surrogates are not detected. |
4387 | | */ |
4388 | 0 | if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { |
4389 | 0 | if(U16_IS_SURROGATE_LEAD(c)) { |
4390 | 0 | getTrail: |
4391 | 0 | if(source<sourceLimit) { |
4392 | | /* test the following code unit */ |
4393 | 0 | UChar trail=*source; |
4394 | 0 | if(U16_IS_TRAIL(trail)) { |
4395 | 0 | ++source; |
4396 | 0 | ++nextSourceIndex; |
4397 | 0 | c=U16_GET_SUPPLEMENTARY(c, trail); |
4398 | 0 | if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { |
4399 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
4400 | 0 | cnv->fromUnicodeStatus=prevLength; /* save the old state */ |
4401 | | /* callback(unassigned) */ |
4402 | 0 | goto unassigned; |
4403 | 0 | } |
4404 | | /* convert this supplementary code point */ |
4405 | | /* exit this condition tree */ |
4406 | 0 | } else { |
4407 | | /* this is an unmatched lead code unit (1st surrogate) */ |
4408 | | /* callback(illegal) */ |
4409 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
4410 | 0 | break; |
4411 | 0 | } |
4412 | 0 | } else { |
4413 | | /* no more input */ |
4414 | 0 | break; |
4415 | 0 | } |
4416 | 0 | } else { |
4417 | | /* this is an unmatched trail code unit (2nd surrogate) */ |
4418 | | /* callback(illegal) */ |
4419 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
4420 | 0 | break; |
4421 | 0 | } |
4422 | 0 | } |
4423 | | |
4424 | | /* convert the Unicode code point in c into codepage bytes */ |
4425 | | |
4426 | | /* |
4427 | | * The basic lookup is a triple-stage compact array (trie) lookup. |
4428 | | * For details see the beginning of this file. |
4429 | | * |
4430 | | * Single-byte codepages are handled with a different data structure |
4431 | | * by _MBCSSingle... functions. |
4432 | | * |
4433 | | * The result consists of a 32-bit value from stage 2 and |
4434 | | * a pointer to as many bytes as are stored per character. |
4435 | | * The pointer points to the character's bytes in stage 3. |
4436 | | * Bits 15..0 of the stage 2 entry contain the stage 3 index |
4437 | | * for that pointer, while bits 31..16 are flags for which of |
4438 | | * the 16 characters in the block are roundtrip-assigned. |
4439 | | * |
4440 | | * For 2-byte and 4-byte codepages, the bytes are stored as uint16_t |
4441 | | * respectively as uint32_t, in the platform encoding. |
4442 | | * For 3-byte codepages, the bytes are always stored in big-endian order. |
4443 | | * |
4444 | | * For EUC encodings that use only either 0x8e or 0x8f as the first |
4445 | | * byte of their longest byte sequences, the first two bytes in |
4446 | | * this third stage indicate with their 7th bits whether these bytes |
4447 | | * are to be written directly or actually need to be preceded by |
4448 | | * one of the two Single-Shift codes. With this, the third stage |
4449 | | * stores one byte fewer per character than the actual maximum length of |
4450 | | * EUC byte sequences. |
4451 | | * |
4452 | | * Other than that, leading zero bytes are removed and the other |
4453 | | * bytes output. A single zero byte may be output if the "assigned" |
4454 | | * bit in stage 2 was on. |
4455 | | * The data structure does not support zero byte output as a fallback, |
4456 | | * and also does not allow output of leading zeros. |
4457 | | */ |
4458 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, c); |
4459 | | |
4460 | | /* get the bytes and the length for the output */ |
4461 | 0 | switch(outputType) { |
4462 | 0 | case MBCS_OUTPUT_2: |
4463 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); |
4464 | 0 | if(value<=0xff) { |
4465 | 0 | length=1; |
4466 | 0 | } else { |
4467 | 0 | length=2; |
4468 | 0 | } |
4469 | 0 | break; |
4470 | 0 | case MBCS_OUTPUT_2_SISO: |
4471 | | /* 1/2-byte stateful with Shift-In/Shift-Out */ |
4472 | | /* |
4473 | | * Save the old state in the converter object |
4474 | | * right here, then change the local prevLength state variable if necessary. |
4475 | | * Then, if this character turns out to be unassigned or a fallback that |
4476 | | * is not taken, the callback code must not save the new state in the converter |
4477 | | * because the new state is for a character that is not output. |
4478 | | * However, the callback must still restore the state from the converter |
4479 | | * in case the callback function changed it for its output. |
4480 | | */ |
4481 | 0 | cnv->fromUnicodeStatus=prevLength; /* save the old state */ |
4482 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); |
4483 | 0 | if(value<=0xff) { |
4484 | 0 | if(value==0 && MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)==0) { |
4485 | | /* no mapping, leave value==0 */ |
4486 | 0 | length=0; |
4487 | 0 | } else if(prevLength<=1) { |
4488 | 0 | length=1; |
4489 | 0 | } else { |
4490 | | /* change from double-byte mode to single-byte */ |
4491 | 0 | if (siLength == 1) { |
4492 | 0 | value|=(uint32_t)siBytes[0]<<8; |
4493 | 0 | length = 2; |
4494 | 0 | } else if (siLength == 2) { |
4495 | 0 | value|=(uint32_t)siBytes[1]<<8; |
4496 | 0 | value|=(uint32_t)siBytes[0]<<16; |
4497 | 0 | length = 3; |
4498 | 0 | } |
4499 | 0 | prevLength=1; |
4500 | 0 | } |
4501 | 0 | } else { |
4502 | 0 | if(prevLength==2) { |
4503 | 0 | length=2; |
4504 | 0 | } else { |
4505 | | /* change from single-byte mode to double-byte */ |
4506 | 0 | if (soLength == 1) { |
4507 | 0 | value|=(uint32_t)soBytes[0]<<16; |
4508 | 0 | length = 3; |
4509 | 0 | } else if (soLength == 2) { |
4510 | 0 | value|=(uint32_t)soBytes[1]<<16; |
4511 | 0 | value|=(uint32_t)soBytes[0]<<24; |
4512 | 0 | length = 4; |
4513 | 0 | } |
4514 | 0 | prevLength=2; |
4515 | 0 | } |
4516 | 0 | } |
4517 | 0 | break; |
4518 | 0 | case MBCS_OUTPUT_DBCS_ONLY: |
4519 | | /* table with single-byte results, but only DBCS mappings used */ |
4520 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); |
4521 | 0 | if(value<=0xff) { |
4522 | | /* no mapping or SBCS result, not taken for DBCS-only */ |
4523 | 0 | value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ |
4524 | 0 | length=0; |
4525 | 0 | } else { |
4526 | 0 | length=2; |
4527 | 0 | } |
4528 | 0 | break; |
4529 | 0 | case MBCS_OUTPUT_3: |
4530 | 0 | p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); |
4531 | 0 | value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; |
4532 | 0 | if(value<=0xff) { |
4533 | 0 | length=1; |
4534 | 0 | } else if(value<=0xffff) { |
4535 | 0 | length=2; |
4536 | 0 | } else { |
4537 | 0 | length=3; |
4538 | 0 | } |
4539 | 0 | break; |
4540 | 0 | case MBCS_OUTPUT_4: |
4541 | 0 | value=MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c); |
4542 | 0 | if(value<=0xff) { |
4543 | 0 | length=1; |
4544 | 0 | } else if(value<=0xffff) { |
4545 | 0 | length=2; |
4546 | 0 | } else if(value<=0xffffff) { |
4547 | 0 | length=3; |
4548 | 0 | } else { |
4549 | 0 | length=4; |
4550 | 0 | } |
4551 | 0 | break; |
4552 | 0 | case MBCS_OUTPUT_3_EUC: |
4553 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); |
4554 | | /* EUC 16-bit fixed-length representation */ |
4555 | 0 | if(value<=0xff) { |
4556 | 0 | length=1; |
4557 | 0 | } else if((value&0x8000)==0) { |
4558 | 0 | value|=0x8e8000; |
4559 | 0 | length=3; |
4560 | 0 | } else if((value&0x80)==0) { |
4561 | 0 | value|=0x8f0080; |
4562 | 0 | length=3; |
4563 | 0 | } else { |
4564 | 0 | length=2; |
4565 | 0 | } |
4566 | 0 | break; |
4567 | 0 | case MBCS_OUTPUT_4_EUC: |
4568 | 0 | p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); |
4569 | 0 | value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; |
4570 | | /* EUC 16-bit fixed-length representation applied to the first two bytes */ |
4571 | 0 | if(value<=0xff) { |
4572 | 0 | length=1; |
4573 | 0 | } else if(value<=0xffff) { |
4574 | 0 | length=2; |
4575 | 0 | } else if((value&0x800000)==0) { |
4576 | 0 | value|=0x8e800000; |
4577 | 0 | length=4; |
4578 | 0 | } else if((value&0x8000)==0) { |
4579 | 0 | value|=0x8f008000; |
4580 | 0 | length=4; |
4581 | 0 | } else { |
4582 | 0 | length=3; |
4583 | 0 | } |
4584 | 0 | break; |
4585 | 0 | default: |
4586 | | /* must not occur */ |
4587 | | /* |
4588 | | * To avoid compiler warnings that value & length may be |
4589 | | * used without having been initialized, we set them here. |
4590 | | * In reality, this is unreachable code. |
4591 | | * Not having a default branch also causes warnings with |
4592 | | * some compilers. |
4593 | | */ |
4594 | 0 | value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ |
4595 | 0 | length=0; |
4596 | 0 | break; |
4597 | 0 | } |
4598 | | |
4599 | | /* is this code point assigned, or do we use fallbacks? */ |
4600 | 0 | if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)!=0 || |
4601 | 0 | (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) |
4602 | 0 | ) { |
4603 | | /* |
4604 | | * We allow a 0 byte output if the "assigned" bit is set for this entry. |
4605 | | * There is no way with this data structure for fallback output |
4606 | | * to be a zero byte. |
4607 | | */ |
4608 | |
|
4609 | 0 | unassigned: |
4610 | | /* try an extension mapping */ |
4611 | 0 | pArgs->source=source; |
4612 | 0 | c=_extFromU(cnv, cnv->sharedData, |
4613 | 0 | c, &source, sourceLimit, |
4614 | 0 | &target, target+targetCapacity, |
4615 | 0 | &offsets, sourceIndex, |
4616 | 0 | pArgs->flush, |
4617 | 0 | pErrorCode); |
4618 | 0 | nextSourceIndex+=(int32_t)(source-pArgs->source); |
4619 | 0 | prevLength=cnv->fromUnicodeStatus; /* restore SISO state */ |
4620 | |
|
4621 | 0 | if(U_FAILURE(*pErrorCode)) { |
4622 | | /* not mappable or buffer overflow */ |
4623 | 0 | break; |
4624 | 0 | } else { |
4625 | | /* a mapping was written to the target, continue */ |
4626 | | |
4627 | | /* recalculate the targetCapacity after an extension mapping */ |
4628 | 0 | targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); |
4629 | | |
4630 | | /* normal end of conversion: prepare for a new character */ |
4631 | 0 | if(offsets!=NULL) { |
4632 | 0 | prevSourceIndex=sourceIndex; |
4633 | 0 | sourceIndex=nextSourceIndex; |
4634 | 0 | } |
4635 | 0 | continue; |
4636 | 0 | } |
4637 | 0 | } |
4638 | 0 | } |
4639 | | |
4640 | | /* write the output character bytes from value and length */ |
4641 | | /* from the first if in the loop we know that targetCapacity>0 */ |
4642 | 0 | if(length<=targetCapacity) { |
4643 | 0 | if(offsets==NULL) { |
4644 | 0 | switch(length) { |
4645 | | /* each branch falls through to the next one */ |
4646 | 0 | case 4: |
4647 | 0 | *target++=(uint8_t)(value>>24); |
4648 | 0 | U_FALLTHROUGH; |
4649 | 0 | case 3: |
4650 | 0 | *target++=(uint8_t)(value>>16); |
4651 | 0 | U_FALLTHROUGH; |
4652 | 0 | case 2: |
4653 | 0 | *target++=(uint8_t)(value>>8); |
4654 | 0 | U_FALLTHROUGH; |
4655 | 0 | case 1: |
4656 | 0 | *target++=(uint8_t)value; |
4657 | 0 | U_FALLTHROUGH; |
4658 | 0 | default: |
4659 | | /* will never occur */ |
4660 | 0 | break; |
4661 | 0 | } |
4662 | 0 | } else { |
4663 | 0 | switch(length) { |
4664 | | /* each branch falls through to the next one */ |
4665 | 0 | case 4: |
4666 | 0 | *target++=(uint8_t)(value>>24); |
4667 | 0 | *offsets++=sourceIndex; |
4668 | 0 | U_FALLTHROUGH; |
4669 | 0 | case 3: |
4670 | 0 | *target++=(uint8_t)(value>>16); |
4671 | 0 | *offsets++=sourceIndex; |
4672 | 0 | U_FALLTHROUGH; |
4673 | 0 | case 2: |
4674 | 0 | *target++=(uint8_t)(value>>8); |
4675 | 0 | *offsets++=sourceIndex; |
4676 | 0 | U_FALLTHROUGH; |
4677 | 0 | case 1: |
4678 | 0 | *target++=(uint8_t)value; |
4679 | 0 | *offsets++=sourceIndex; |
4680 | 0 | U_FALLTHROUGH; |
4681 | 0 | default: |
4682 | | /* will never occur */ |
4683 | 0 | break; |
4684 | 0 | } |
4685 | 0 | } |
4686 | 0 | targetCapacity-=length; |
4687 | 0 | } else { |
4688 | 0 | uint8_t *charErrorBuffer; |
4689 | | |
4690 | | /* |
4691 | | * We actually do this backwards here: |
4692 | | * In order to save an intermediate variable, we output |
4693 | | * first to the overflow buffer what does not fit into the |
4694 | | * regular target. |
4695 | | */ |
4696 | | /* we know that 1<=targetCapacity<length<=4 */ |
4697 | 0 | length-=targetCapacity; |
4698 | 0 | charErrorBuffer=(uint8_t *)cnv->charErrorBuffer; |
4699 | 0 | switch(length) { |
4700 | | /* each branch falls through to the next one */ |
4701 | 0 | case 3: |
4702 | 0 | *charErrorBuffer++=(uint8_t)(value>>16); |
4703 | 0 | U_FALLTHROUGH; |
4704 | 0 | case 2: |
4705 | 0 | *charErrorBuffer++=(uint8_t)(value>>8); |
4706 | 0 | U_FALLTHROUGH; |
4707 | 0 | case 1: |
4708 | 0 | *charErrorBuffer=(uint8_t)value; |
4709 | 0 | U_FALLTHROUGH; |
4710 | 0 | default: |
4711 | | /* will never occur */ |
4712 | 0 | break; |
4713 | 0 | } |
4714 | 0 | cnv->charErrorBufferLength=(int8_t)length; |
4715 | | |
4716 | | /* now output what fits into the regular target */ |
4717 | 0 | value>>=8*length; /* length was reduced by targetCapacity */ |
4718 | 0 | switch(targetCapacity) { |
4719 | | /* each branch falls through to the next one */ |
4720 | 0 | case 3: |
4721 | 0 | *target++=(uint8_t)(value>>16); |
4722 | 0 | if(offsets!=NULL) { |
4723 | 0 | *offsets++=sourceIndex; |
4724 | 0 | } |
4725 | 0 | U_FALLTHROUGH; |
4726 | 0 | case 2: |
4727 | 0 | *target++=(uint8_t)(value>>8); |
4728 | 0 | if(offsets!=NULL) { |
4729 | 0 | *offsets++=sourceIndex; |
4730 | 0 | } |
4731 | 0 | U_FALLTHROUGH; |
4732 | 0 | case 1: |
4733 | 0 | *target++=(uint8_t)value; |
4734 | 0 | if(offsets!=NULL) { |
4735 | 0 | *offsets++=sourceIndex; |
4736 | 0 | } |
4737 | 0 | U_FALLTHROUGH; |
4738 | 0 | default: |
4739 | | /* will never occur */ |
4740 | 0 | break; |
4741 | 0 | } |
4742 | | |
4743 | | /* target overflow */ |
4744 | 0 | targetCapacity=0; |
4745 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
4746 | 0 | c=0; |
4747 | 0 | break; |
4748 | 0 | } |
4749 | | |
4750 | | /* normal end of conversion: prepare for a new character */ |
4751 | 0 | c=0; |
4752 | 0 | if(offsets!=NULL) { |
4753 | 0 | prevSourceIndex=sourceIndex; |
4754 | 0 | sourceIndex=nextSourceIndex; |
4755 | 0 | } |
4756 | 0 | continue; |
4757 | 0 | } else { |
4758 | | /* target is full */ |
4759 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
4760 | 0 | break; |
4761 | 0 | } |
4762 | 0 | } |
4763 | | |
4764 | | /* |
4765 | | * the end of the input stream and detection of truncated input |
4766 | | * are handled by the framework, but for EBCDIC_STATEFUL conversion |
4767 | | * we need to emit an SI at the very end |
4768 | | * |
4769 | | * conditions: |
4770 | | * successful |
4771 | | * EBCDIC_STATEFUL in DBCS mode |
4772 | | * end of input and no truncated input |
4773 | | */ |
4774 | 0 | if( U_SUCCESS(*pErrorCode) && |
4775 | 0 | outputType==MBCS_OUTPUT_2_SISO && prevLength==2 && |
4776 | 0 | pArgs->flush && source>=sourceLimit && c==0 |
4777 | 0 | ) { |
4778 | | /* EBCDIC_STATEFUL ending with DBCS: emit an SI to return the output stream to SBCS */ |
4779 | 0 | if(targetCapacity>0) { |
4780 | 0 | *target++=(uint8_t)siBytes[0]; |
4781 | 0 | if (siLength == 2) { |
4782 | 0 | if (targetCapacity<2) { |
4783 | 0 | cnv->charErrorBuffer[0]=(uint8_t)siBytes[1]; |
4784 | 0 | cnv->charErrorBufferLength=1; |
4785 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
4786 | 0 | } else { |
4787 | 0 | *target++=(uint8_t)siBytes[1]; |
4788 | 0 | } |
4789 | 0 | } |
4790 | 0 | if(offsets!=NULL) { |
4791 | | /* set the last source character's index (sourceIndex points at sourceLimit now) */ |
4792 | 0 | *offsets++=prevSourceIndex; |
4793 | 0 | } |
4794 | 0 | } else { |
4795 | | /* target is full */ |
4796 | 0 | cnv->charErrorBuffer[0]=(uint8_t)siBytes[0]; |
4797 | 0 | if (siLength == 2) { |
4798 | 0 | cnv->charErrorBuffer[1]=(uint8_t)siBytes[1]; |
4799 | 0 | } |
4800 | 0 | cnv->charErrorBufferLength=siLength; |
4801 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
4802 | 0 | } |
4803 | 0 | prevLength=1; /* we switched into SBCS */ |
4804 | 0 | } |
4805 | | |
4806 | | /* set the converter state back into UConverter */ |
4807 | 0 | cnv->fromUChar32=c; |
4808 | 0 | cnv->fromUnicodeStatus=prevLength; |
4809 | | |
4810 | | /* write back the updated pointers */ |
4811 | 0 | pArgs->source=source; |
4812 | 0 | pArgs->target=(char *)target; |
4813 | 0 | pArgs->offsets=offsets; |
4814 | 0 | } |
4815 | | |
4816 | | /* |
4817 | | * This is another simple conversion function for internal use by other |
4818 | | * conversion implementations. |
4819 | | * It does not use the converter state nor call callbacks. |
4820 | | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
4821 | | * It handles conversion extensions but not GB 18030. |
4822 | | * |
4823 | | * It converts one single Unicode code point into codepage bytes, encoded |
4824 | | * as one 32-bit value. The function returns the number of bytes in *pValue: |
4825 | | * 1..4 the number of bytes in *pValue |
4826 | | * 0 unassigned (*pValue undefined) |
4827 | | * -1 illegal (currently not used, *pValue undefined) |
4828 | | * |
4829 | | * *pValue will contain the resulting bytes with the last byte in bits 7..0, |
4830 | | * the second to last byte in bits 15..8, etc. |
4831 | | * Currently, the function assumes but does not check that 0<=c<=0x10ffff. |
4832 | | */ |
4833 | | U_CFUNC int32_t |
4834 | | ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, |
4835 | | UChar32 c, uint32_t *pValue, |
4836 | 0 | UBool useFallback) { |
4837 | 0 | const int32_t *cx; |
4838 | 0 | const uint16_t *table; |
4839 | | #if 0 |
4840 | | /* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ |
4841 | | const uint8_t *p; |
4842 | | #endif |
4843 | 0 | uint32_t stage2Entry; |
4844 | 0 | uint32_t value; |
4845 | 0 | int32_t length; |
4846 | | |
4847 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
4848 | 0 | if(c<=0xffff || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { |
4849 | 0 | table=sharedData->mbcs.fromUnicodeTable; |
4850 | | |
4851 | | /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ |
4852 | 0 | if(sharedData->mbcs.outputType==MBCS_OUTPUT_1) { |
4853 | 0 | value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); |
4854 | | /* is this code point assigned, or do we use fallbacks? */ |
4855 | 0 | if(useFallback ? value>=0x800 : value>=0xc00) { |
4856 | 0 | *pValue=value&0xff; |
4857 | 0 | return 1; |
4858 | 0 | } |
4859 | 0 | } else /* outputType!=MBCS_OUTPUT_1 */ { |
4860 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, c); |
4861 | | |
4862 | | /* get the bytes and the length for the output */ |
4863 | 0 | switch(sharedData->mbcs.outputType) { |
4864 | 0 | case MBCS_OUTPUT_2: |
4865 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); |
4866 | 0 | if(value<=0xff) { |
4867 | 0 | length=1; |
4868 | 0 | } else { |
4869 | 0 | length=2; |
4870 | 0 | } |
4871 | 0 | break; |
4872 | | #if 0 |
4873 | | /* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ |
4874 | | case MBCS_OUTPUT_DBCS_ONLY: |
4875 | | /* table with single-byte results, but only DBCS mappings used */ |
4876 | | value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); |
4877 | | if(value<=0xff) { |
4878 | | /* no mapping or SBCS result, not taken for DBCS-only */ |
4879 | | value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ |
4880 | | length=0; |
4881 | | } else { |
4882 | | length=2; |
4883 | | } |
4884 | | break; |
4885 | | case MBCS_OUTPUT_3: |
4886 | | p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); |
4887 | | value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; |
4888 | | if(value<=0xff) { |
4889 | | length=1; |
4890 | | } else if(value<=0xffff) { |
4891 | | length=2; |
4892 | | } else { |
4893 | | length=3; |
4894 | | } |
4895 | | break; |
4896 | | case MBCS_OUTPUT_4: |
4897 | | value=MBCS_VALUE_4_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); |
4898 | | if(value<=0xff) { |
4899 | | length=1; |
4900 | | } else if(value<=0xffff) { |
4901 | | length=2; |
4902 | | } else if(value<=0xffffff) { |
4903 | | length=3; |
4904 | | } else { |
4905 | | length=4; |
4906 | | } |
4907 | | break; |
4908 | | case MBCS_OUTPUT_3_EUC: |
4909 | | value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); |
4910 | | /* EUC 16-bit fixed-length representation */ |
4911 | | if(value<=0xff) { |
4912 | | length=1; |
4913 | | } else if((value&0x8000)==0) { |
4914 | | value|=0x8e8000; |
4915 | | length=3; |
4916 | | } else if((value&0x80)==0) { |
4917 | | value|=0x8f0080; |
4918 | | length=3; |
4919 | | } else { |
4920 | | length=2; |
4921 | | } |
4922 | | break; |
4923 | | case MBCS_OUTPUT_4_EUC: |
4924 | | p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); |
4925 | | value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; |
4926 | | /* EUC 16-bit fixed-length representation applied to the first two bytes */ |
4927 | | if(value<=0xff) { |
4928 | | length=1; |
4929 | | } else if(value<=0xffff) { |
4930 | | length=2; |
4931 | | } else if((value&0x800000)==0) { |
4932 | | value|=0x8e800000; |
4933 | | length=4; |
4934 | | } else if((value&0x8000)==0) { |
4935 | | value|=0x8f008000; |
4936 | | length=4; |
4937 | | } else { |
4938 | | length=3; |
4939 | | } |
4940 | | break; |
4941 | | #endif |
4942 | 0 | default: |
4943 | | /* must not occur */ |
4944 | 0 | return -1; |
4945 | 0 | } |
4946 | | |
4947 | | /* is this code point assigned, or do we use fallbacks? */ |
4948 | 0 | if( MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || |
4949 | 0 | (FROM_U_USE_FALLBACK(useFallback, c) && value!=0) |
4950 | 0 | ) { |
4951 | | /* |
4952 | | * We allow a 0 byte output if the "assigned" bit is set for this entry. |
4953 | | * There is no way with this data structure for fallback output |
4954 | | * to be a zero byte. |
4955 | | */ |
4956 | | /* assigned */ |
4957 | 0 | *pValue=value; |
4958 | 0 | return length; |
4959 | 0 | } |
4960 | 0 | } |
4961 | 0 | } |
4962 | | |
4963 | 0 | cx=sharedData->mbcs.extIndexes; |
4964 | 0 | if(cx!=NULL) { |
4965 | 0 | length=ucnv_extSimpleMatchFromU(cx, c, pValue, useFallback); |
4966 | 0 | return length>=0 ? length : -length; /* return abs(length); */ |
4967 | 0 | } |
4968 | | |
4969 | | /* unassigned */ |
4970 | 0 | return 0; |
4971 | 0 | } |
4972 | | |
4973 | | |
4974 | | #if 0 |
4975 | | /* |
4976 | | * This function has been moved to ucnv2022.c for inlining. |
4977 | | * This implementation is here only for documentation purposes |
4978 | | */ |
4979 | | |
4980 | | /** |
4981 | | * This version of ucnv_MBCSFromUChar32() is optimized for single-byte codepages. |
4982 | | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
4983 | | * It does not handle conversion extensions (_extFromU()). |
4984 | | * |
4985 | | * It returns the codepage byte for the code point, or -1 if it is unassigned. |
4986 | | */ |
4987 | | U_CFUNC int32_t |
4988 | | ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, |
4989 | | UChar32 c, |
4990 | | UBool useFallback) { |
4991 | | const uint16_t *table; |
4992 | | int32_t value; |
4993 | | |
4994 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
4995 | | if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { |
4996 | | return -1; |
4997 | | } |
4998 | | |
4999 | | /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ |
5000 | | table=sharedData->mbcs.fromUnicodeTable; |
5001 | | |
5002 | | /* get the byte for the output */ |
5003 | | value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); |
5004 | | /* is this code point assigned, or do we use fallbacks? */ |
5005 | | if(useFallback ? value>=0x800 : value>=0xc00) { |
5006 | | return value&0xff; |
5007 | | } else { |
5008 | | return -1; |
5009 | | } |
5010 | | } |
5011 | | #endif |
5012 | | |
5013 | | /* MBCS-from-UTF-8 conversion functions ------------------------------------- */ |
5014 | | |
5015 | | /* offsets for n-byte UTF-8 sequences that were calculated with ((lead<<6)+trail)<<6+trail... */ |
5016 | | static const UChar32 |
5017 | | utf8_offsets[5]={ 0, 0, 0x3080, 0xE2080, 0x3C82080 }; |
5018 | | |
5019 | | static void U_CALLCONV |
5020 | | ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, |
5021 | | UConverterToUnicodeArgs *pToUArgs, |
5022 | 0 | UErrorCode *pErrorCode) { |
5023 | 0 | UConverter *utf8, *cnv; |
5024 | 0 | const uint8_t *source, *sourceLimit; |
5025 | 0 | uint8_t *target; |
5026 | 0 | int32_t targetCapacity; |
5027 | |
|
5028 | 0 | const uint16_t *table, *sbcsIndex; |
5029 | 0 | const uint16_t *results; |
5030 | |
|
5031 | 0 | int8_t oldToULength, toULength, toULimit; |
5032 | |
|
5033 | 0 | UChar32 c; |
5034 | 0 | uint8_t b, t1, t2; |
5035 | |
|
5036 | 0 | uint32_t asciiRoundtrips; |
5037 | 0 | uint16_t value, minValue = 0; |
5038 | 0 | UBool hasSupplementary; |
5039 | | |
5040 | | /* set up the local pointers */ |
5041 | 0 | utf8=pToUArgs->converter; |
5042 | 0 | cnv=pFromUArgs->converter; |
5043 | 0 | source=(uint8_t *)pToUArgs->source; |
5044 | 0 | sourceLimit=(uint8_t *)pToUArgs->sourceLimit; |
5045 | 0 | target=(uint8_t *)pFromUArgs->target; |
5046 | 0 | targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); |
5047 | |
|
5048 | 0 | table=cnv->sharedData->mbcs.fromUnicodeTable; |
5049 | 0 | sbcsIndex=cnv->sharedData->mbcs.sbcsIndex; |
5050 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
5051 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; |
5052 | 0 | } else { |
5053 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; |
5054 | 0 | } |
5055 | 0 | asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; |
5056 | |
|
5057 | 0 | if(cnv->useFallback) { |
5058 | | /* use all roundtrip and fallback results */ |
5059 | 0 | minValue=0x800; |
5060 | 0 | } else { |
5061 | | /* use only roundtrips and fallbacks from private-use characters */ |
5062 | 0 | minValue=0xc00; |
5063 | 0 | } |
5064 | 0 | hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); |
5065 | | |
5066 | | /* get the converter state from the UTF-8 UConverter */ |
5067 | 0 | if(utf8->toULength > 0) { |
5068 | 0 | toULength=oldToULength=utf8->toULength; |
5069 | 0 | toULimit=(int8_t)utf8->mode; |
5070 | 0 | c=(UChar32)utf8->toUnicodeStatus; |
5071 | 0 | } else { |
5072 | 0 | toULength=oldToULength=toULimit=0; |
5073 | 0 | c = 0; |
5074 | 0 | } |
5075 | | |
5076 | | // The conversion loop checks source<sourceLimit only once per 1/2/3-byte character. |
5077 | | // If the buffer ends with a truncated 2- or 3-byte sequence, |
5078 | | // then we reduce the sourceLimit to before that, |
5079 | | // and collect the remaining bytes after the conversion loop. |
5080 | 0 | { |
5081 | | // Do not go back into the bytes that will be read for finishing a partial |
5082 | | // sequence from the previous buffer. |
5083 | 0 | int32_t length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); |
5084 | 0 | if(length>0) { |
5085 | 0 | uint8_t b1=*(sourceLimit-1); |
5086 | 0 | if(U8_IS_SINGLE(b1)) { |
5087 | | // common ASCII character |
5088 | 0 | } else if(U8_IS_TRAIL(b1) && length>=2) { |
5089 | 0 | uint8_t b2=*(sourceLimit-2); |
5090 | 0 | if(0xe0<=b2 && b2<0xf0 && U8_IS_VALID_LEAD3_AND_T1(b2, b1)) { |
5091 | | // truncated 3-byte sequence |
5092 | 0 | sourceLimit-=2; |
5093 | 0 | } |
5094 | 0 | } else if(0xc2<=b1 && b1<0xf0) { |
5095 | | // truncated 2- or 3-byte sequence |
5096 | 0 | --sourceLimit; |
5097 | 0 | } |
5098 | 0 | } |
5099 | 0 | } |
5100 | |
|
5101 | 0 | if(c!=0 && targetCapacity>0) { |
5102 | 0 | utf8->toUnicodeStatus=0; |
5103 | 0 | utf8->toULength=0; |
5104 | 0 | goto moreBytes; |
5105 | | /* |
5106 | | * Note: We could avoid the goto by duplicating some of the moreBytes |
5107 | | * code, but only up to the point of collecting a complete UTF-8 |
5108 | | * sequence; then recurse for the toUBytes[toULength] |
5109 | | * and then continue with normal conversion. |
5110 | | * |
5111 | | * If so, move this code to just after initializing the minimum |
5112 | | * set of local variables for reading the UTF-8 input |
5113 | | * (utf8, source, target, limits but not cnv, table, minValue, etc.). |
5114 | | * |
5115 | | * Potential advantages: |
5116 | | * - avoid the goto |
5117 | | * - oldToULength could become a local variable in just those code blocks |
5118 | | * that deal with buffer boundaries |
5119 | | * - possibly faster if the goto prevents some compiler optimizations |
5120 | | * (this would need measuring to confirm) |
5121 | | * Disadvantage: |
5122 | | * - code duplication |
5123 | | */ |
5124 | 0 | } |
5125 | | |
5126 | | /* conversion loop */ |
5127 | 0 | while(source<sourceLimit) { |
5128 | 0 | if(targetCapacity>0) { |
5129 | 0 | b=*source++; |
5130 | 0 | if(U8_IS_SINGLE(b)) { |
5131 | | /* convert ASCII */ |
5132 | 0 | if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { |
5133 | 0 | *target++=(uint8_t)b; |
5134 | 0 | --targetCapacity; |
5135 | 0 | continue; |
5136 | 0 | } else { |
5137 | 0 | c=b; |
5138 | 0 | value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, 0, c); |
5139 | 0 | } |
5140 | 0 | } else { |
5141 | 0 | if(b<0xe0) { |
5142 | 0 | if( /* handle U+0080..U+07FF inline */ |
5143 | 0 | b>=0xc2 && |
5144 | 0 | (t1=(uint8_t)(*source-0x80)) <= 0x3f |
5145 | 0 | ) { |
5146 | 0 | c=b&0x1f; |
5147 | 0 | ++source; |
5148 | 0 | value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t1); |
5149 | 0 | if(value>=minValue) { |
5150 | 0 | *target++=(uint8_t)value; |
5151 | 0 | --targetCapacity; |
5152 | 0 | continue; |
5153 | 0 | } else { |
5154 | 0 | c=(c<<6)|t1; |
5155 | 0 | } |
5156 | 0 | } else { |
5157 | 0 | c=-1; |
5158 | 0 | } |
5159 | 0 | } else if(b==0xe0) { |
5160 | 0 | if( /* handle U+0800..U+0FFF inline */ |
5161 | 0 | (t1=(uint8_t)(source[0]-0x80)) <= 0x3f && t1 >= 0x20 && |
5162 | 0 | (t2=(uint8_t)(source[1]-0x80)) <= 0x3f |
5163 | 0 | ) { |
5164 | 0 | c=t1; |
5165 | 0 | source+=2; |
5166 | 0 | value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t2); |
5167 | 0 | if(value>=minValue) { |
5168 | 0 | *target++=(uint8_t)value; |
5169 | 0 | --targetCapacity; |
5170 | 0 | continue; |
5171 | 0 | } else { |
5172 | 0 | c=(c<<6)|t2; |
5173 | 0 | } |
5174 | 0 | } else { |
5175 | 0 | c=-1; |
5176 | 0 | } |
5177 | 0 | } else { |
5178 | 0 | c=-1; |
5179 | 0 | } |
5180 | | |
5181 | 0 | if(c<0) { |
5182 | | /* handle "complicated" and error cases, and continuing partial characters */ |
5183 | 0 | oldToULength=0; |
5184 | 0 | toULength=1; |
5185 | 0 | toULimit=U8_COUNT_BYTES_NON_ASCII(b); |
5186 | 0 | c=b; |
5187 | 0 | moreBytes: |
5188 | 0 | while(toULength<toULimit) { |
5189 | | /* |
5190 | | * The sourceLimit may have been adjusted before the conversion loop |
5191 | | * to stop before a truncated sequence. |
5192 | | * Here we need to use the real limit in case we have two truncated |
5193 | | * sequences at the end. |
5194 | | * See ticket #7492. |
5195 | | */ |
5196 | 0 | if(source<(uint8_t *)pToUArgs->sourceLimit) { |
5197 | 0 | b=*source; |
5198 | 0 | if(icu::UTF8::isValidTrail(c, b, toULength, toULimit)) { |
5199 | 0 | ++source; |
5200 | 0 | ++toULength; |
5201 | 0 | c=(c<<6)+b; |
5202 | 0 | } else { |
5203 | 0 | break; /* sequence too short, stop with toULength<toULimit */ |
5204 | 0 | } |
5205 | 0 | } else { |
5206 | | /* store the partial UTF-8 character, compatible with the regular UTF-8 converter */ |
5207 | 0 | source-=(toULength-oldToULength); |
5208 | 0 | while(oldToULength<toULength) { |
5209 | 0 | utf8->toUBytes[oldToULength++]=*source++; |
5210 | 0 | } |
5211 | 0 | utf8->toUnicodeStatus=c; |
5212 | 0 | utf8->toULength=toULength; |
5213 | 0 | utf8->mode=toULimit; |
5214 | 0 | pToUArgs->source=(char *)source; |
5215 | 0 | pFromUArgs->target=(char *)target; |
5216 | 0 | return; |
5217 | 0 | } |
5218 | 0 | } |
5219 | | |
5220 | 0 | if(toULength==toULimit) { |
5221 | 0 | c-=utf8_offsets[toULength]; |
5222 | 0 | if(toULength<=3) { /* BMP */ |
5223 | 0 | value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
5224 | 0 | } else { |
5225 | | /* supplementary code point */ |
5226 | 0 | if(!hasSupplementary) { |
5227 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
5228 | 0 | value=0; |
5229 | 0 | } else { |
5230 | 0 | value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); |
5231 | 0 | } |
5232 | 0 | } |
5233 | 0 | } else { |
5234 | | /* error handling: illegal UTF-8 byte sequence */ |
5235 | 0 | source-=(toULength-oldToULength); |
5236 | 0 | while(oldToULength<toULength) { |
5237 | 0 | utf8->toUBytes[oldToULength++]=*source++; |
5238 | 0 | } |
5239 | 0 | utf8->toULength=toULength; |
5240 | 0 | pToUArgs->source=(char *)source; |
5241 | 0 | pFromUArgs->target=(char *)target; |
5242 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
5243 | 0 | return; |
5244 | 0 | } |
5245 | 0 | } |
5246 | 0 | } |
5247 | | |
5248 | 0 | if(value>=minValue) { |
5249 | | /* output the mapping for c */ |
5250 | 0 | *target++=(uint8_t)value; |
5251 | 0 | --targetCapacity; |
5252 | 0 | } else { |
5253 | | /* value<minValue means c is unassigned (unmappable) */ |
5254 | | /* |
5255 | | * Try an extension mapping. |
5256 | | * Pass in no source because we don't have UTF-16 input. |
5257 | | * If we have a partial match on c, we will return and revert |
5258 | | * to UTF-8->UTF-16->charset conversion. |
5259 | | */ |
5260 | 0 | static const UChar nul=0; |
5261 | 0 | const UChar *noSource=&nul; |
5262 | 0 | c=_extFromU(cnv, cnv->sharedData, |
5263 | 0 | c, &noSource, noSource, |
5264 | 0 | &target, target+targetCapacity, |
5265 | 0 | NULL, -1, |
5266 | 0 | pFromUArgs->flush, |
5267 | 0 | pErrorCode); |
5268 | |
|
5269 | 0 | if(U_FAILURE(*pErrorCode)) { |
5270 | | /* not mappable or buffer overflow */ |
5271 | 0 | cnv->fromUChar32=c; |
5272 | 0 | break; |
5273 | 0 | } else if(cnv->preFromUFirstCP>=0) { |
5274 | | /* |
5275 | | * Partial match, return and revert to pivoting. |
5276 | | * In normal from-UTF-16 conversion, we would just continue |
5277 | | * but then exit the loop because the extension match would |
5278 | | * have consumed the source. |
5279 | | */ |
5280 | 0 | *pErrorCode=U_USING_DEFAULT_WARNING; |
5281 | 0 | break; |
5282 | 0 | } else { |
5283 | | /* a mapping was written to the target, continue */ |
5284 | | |
5285 | | /* recalculate the targetCapacity after an extension mapping */ |
5286 | 0 | targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); |
5287 | 0 | } |
5288 | 0 | } |
5289 | 0 | } else { |
5290 | | /* target is full */ |
5291 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
5292 | 0 | break; |
5293 | 0 | } |
5294 | 0 | } |
5295 | | |
5296 | | /* |
5297 | | * The sourceLimit may have been adjusted before the conversion loop |
5298 | | * to stop before a truncated sequence. |
5299 | | * If so, then collect the truncated sequence now. |
5300 | | */ |
5301 | 0 | if(U_SUCCESS(*pErrorCode) && |
5302 | 0 | cnv->preFromUFirstCP<0 && |
5303 | 0 | source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { |
5304 | 0 | c=utf8->toUBytes[0]=b=*source++; |
5305 | 0 | toULength=1; |
5306 | 0 | toULimit=U8_COUNT_BYTES(b); |
5307 | 0 | while(source<sourceLimit) { |
5308 | 0 | utf8->toUBytes[toULength++]=b=*source++; |
5309 | 0 | c=(c<<6)+b; |
5310 | 0 | } |
5311 | 0 | utf8->toUnicodeStatus=c; |
5312 | 0 | utf8->toULength=toULength; |
5313 | 0 | utf8->mode=toULimit; |
5314 | 0 | } |
5315 | | |
5316 | | /* write back the updated pointers */ |
5317 | 0 | pToUArgs->source=(char *)source; |
5318 | 0 | pFromUArgs->target=(char *)target; |
5319 | 0 | } |
5320 | | |
5321 | | static void U_CALLCONV |
5322 | | ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, |
5323 | | UConverterToUnicodeArgs *pToUArgs, |
5324 | 0 | UErrorCode *pErrorCode) { |
5325 | 0 | UConverter *utf8, *cnv; |
5326 | 0 | const uint8_t *source, *sourceLimit; |
5327 | 0 | uint8_t *target; |
5328 | 0 | int32_t targetCapacity; |
5329 | |
|
5330 | 0 | const uint16_t *table, *mbcsIndex; |
5331 | 0 | const uint16_t *results; |
5332 | |
|
5333 | 0 | int8_t oldToULength, toULength, toULimit; |
5334 | |
|
5335 | 0 | UChar32 c; |
5336 | 0 | uint8_t b, t1, t2; |
5337 | |
|
5338 | 0 | uint32_t stage2Entry; |
5339 | 0 | uint32_t asciiRoundtrips; |
5340 | 0 | uint16_t value = 0; |
5341 | 0 | UBool hasSupplementary; |
5342 | | |
5343 | | /* set up the local pointers */ |
5344 | 0 | utf8=pToUArgs->converter; |
5345 | 0 | cnv=pFromUArgs->converter; |
5346 | 0 | source=(uint8_t *)pToUArgs->source; |
5347 | 0 | sourceLimit=(uint8_t *)pToUArgs->sourceLimit; |
5348 | 0 | target=(uint8_t *)pFromUArgs->target; |
5349 | 0 | targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); |
5350 | |
|
5351 | 0 | table=cnv->sharedData->mbcs.fromUnicodeTable; |
5352 | 0 | mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; |
5353 | 0 | if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { |
5354 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; |
5355 | 0 | } else { |
5356 | 0 | results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; |
5357 | 0 | } |
5358 | 0 | asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; |
5359 | |
|
5360 | 0 | hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); |
5361 | | |
5362 | | /* get the converter state from the UTF-8 UConverter */ |
5363 | 0 | if(utf8->toULength > 0) { |
5364 | 0 | toULength=oldToULength=utf8->toULength; |
5365 | 0 | toULimit=(int8_t)utf8->mode; |
5366 | 0 | c=(UChar32)utf8->toUnicodeStatus; |
5367 | 0 | } else { |
5368 | 0 | toULength=oldToULength=toULimit=0; |
5369 | 0 | c = 0; |
5370 | 0 | } |
5371 | | |
5372 | | // The conversion loop checks source<sourceLimit only once per 1/2/3-byte character. |
5373 | | // If the buffer ends with a truncated 2- or 3-byte sequence, |
5374 | | // then we reduce the sourceLimit to before that, |
5375 | | // and collect the remaining bytes after the conversion loop. |
5376 | 0 | { |
5377 | | // Do not go back into the bytes that will be read for finishing a partial |
5378 | | // sequence from the previous buffer. |
5379 | 0 | int32_t length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); |
5380 | 0 | if(length>0) { |
5381 | 0 | uint8_t b1=*(sourceLimit-1); |
5382 | 0 | if(U8_IS_SINGLE(b1)) { |
5383 | | // common ASCII character |
5384 | 0 | } else if(U8_IS_TRAIL(b1) && length>=2) { |
5385 | 0 | uint8_t b2=*(sourceLimit-2); |
5386 | 0 | if(0xe0<=b2 && b2<0xf0 && U8_IS_VALID_LEAD3_AND_T1(b2, b1)) { |
5387 | | // truncated 3-byte sequence |
5388 | 0 | sourceLimit-=2; |
5389 | 0 | } |
5390 | 0 | } else if(0xc2<=b1 && b1<0xf0) { |
5391 | | // truncated 2- or 3-byte sequence |
5392 | 0 | --sourceLimit; |
5393 | 0 | } |
5394 | 0 | } |
5395 | 0 | } |
5396 | |
|
5397 | 0 | if(c!=0 && targetCapacity>0) { |
5398 | 0 | utf8->toUnicodeStatus=0; |
5399 | 0 | utf8->toULength=0; |
5400 | 0 | goto moreBytes; |
5401 | | /* See note in ucnv_SBCSFromUTF8() about this goto. */ |
5402 | 0 | } |
5403 | | |
5404 | | /* conversion loop */ |
5405 | 0 | while(source<sourceLimit) { |
5406 | 0 | if(targetCapacity>0) { |
5407 | 0 | b=*source++; |
5408 | 0 | if(U8_IS_SINGLE(b)) { |
5409 | | /* convert ASCII */ |
5410 | 0 | if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { |
5411 | 0 | *target++=b; |
5412 | 0 | --targetCapacity; |
5413 | 0 | continue; |
5414 | 0 | } else { |
5415 | 0 | value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, 0, b); |
5416 | 0 | if(value==0) { |
5417 | 0 | c=b; |
5418 | 0 | goto unassigned; |
5419 | 0 | } |
5420 | 0 | } |
5421 | 0 | } else { |
5422 | 0 | if(b>=0xe0) { |
5423 | 0 | if( /* handle U+0800..U+D7FF inline */ |
5424 | 0 | b<=0xed && // do not assume maxFastUChar>0xd7ff |
5425 | 0 | U8_IS_VALID_LEAD3_AND_T1(b, t1=source[0]) && |
5426 | 0 | (t2=(uint8_t)(source[1]-0x80)) <= 0x3f |
5427 | 0 | ) { |
5428 | 0 | c=((b&0xf)<<6)|(t1&0x3f); |
5429 | 0 | source+=2; |
5430 | 0 | value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t2); |
5431 | 0 | if(value==0) { |
5432 | 0 | c=(c<<6)|t2; |
5433 | 0 | goto unassigned; |
5434 | 0 | } |
5435 | 0 | } else { |
5436 | 0 | c=-1; |
5437 | 0 | } |
5438 | 0 | } else { |
5439 | 0 | if( /* handle U+0080..U+07FF inline */ |
5440 | 0 | b>=0xc2 && |
5441 | 0 | (t1=(uint8_t)(*source-0x80)) <= 0x3f |
5442 | 0 | ) { |
5443 | 0 | c=b&0x1f; |
5444 | 0 | ++source; |
5445 | 0 | value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t1); |
5446 | 0 | if(value==0) { |
5447 | 0 | c=(c<<6)|t1; |
5448 | 0 | goto unassigned; |
5449 | 0 | } |
5450 | 0 | } else { |
5451 | 0 | c=-1; |
5452 | 0 | } |
5453 | 0 | } |
5454 | | |
5455 | 0 | if(c<0) { |
5456 | | /* handle "complicated" and error cases, and continuing partial characters */ |
5457 | 0 | oldToULength=0; |
5458 | 0 | toULength=1; |
5459 | 0 | toULimit=U8_COUNT_BYTES_NON_ASCII(b); |
5460 | 0 | c=b; |
5461 | 0 | moreBytes: |
5462 | 0 | while(toULength<toULimit) { |
5463 | | /* |
5464 | | * The sourceLimit may have been adjusted before the conversion loop |
5465 | | * to stop before a truncated sequence. |
5466 | | * Here we need to use the real limit in case we have two truncated |
5467 | | * sequences at the end. |
5468 | | * See ticket #7492. |
5469 | | */ |
5470 | 0 | if(source<(uint8_t *)pToUArgs->sourceLimit) { |
5471 | 0 | b=*source; |
5472 | 0 | if(icu::UTF8::isValidTrail(c, b, toULength, toULimit)) { |
5473 | 0 | ++source; |
5474 | 0 | ++toULength; |
5475 | 0 | c=(c<<6)+b; |
5476 | 0 | } else { |
5477 | 0 | break; /* sequence too short, stop with toULength<toULimit */ |
5478 | 0 | } |
5479 | 0 | } else { |
5480 | | /* store the partial UTF-8 character, compatible with the regular UTF-8 converter */ |
5481 | 0 | source-=(toULength-oldToULength); |
5482 | 0 | while(oldToULength<toULength) { |
5483 | 0 | utf8->toUBytes[oldToULength++]=*source++; |
5484 | 0 | } |
5485 | 0 | utf8->toUnicodeStatus=c; |
5486 | 0 | utf8->toULength=toULength; |
5487 | 0 | utf8->mode=toULimit; |
5488 | 0 | pToUArgs->source=(char *)source; |
5489 | 0 | pFromUArgs->target=(char *)target; |
5490 | 0 | return; |
5491 | 0 | } |
5492 | 0 | } |
5493 | | |
5494 | 0 | if(toULength==toULimit) { |
5495 | 0 | c-=utf8_offsets[toULength]; |
5496 | 0 | if(toULength<=3) { /* BMP */ |
5497 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, c); |
5498 | 0 | } else { |
5499 | | /* supplementary code point */ |
5500 | 0 | if(!hasSupplementary) { |
5501 | | /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ |
5502 | 0 | stage2Entry=0; |
5503 | 0 | } else { |
5504 | 0 | stage2Entry=MBCS_STAGE_2_FROM_U(table, c); |
5505 | 0 | } |
5506 | 0 | } |
5507 | 0 | } else { |
5508 | | /* error handling: illegal UTF-8 byte sequence */ |
5509 | 0 | source-=(toULength-oldToULength); |
5510 | 0 | while(oldToULength<toULength) { |
5511 | 0 | utf8->toUBytes[oldToULength++]=*source++; |
5512 | 0 | } |
5513 | 0 | utf8->toULength=toULength; |
5514 | 0 | pToUArgs->source=(char *)source; |
5515 | 0 | pFromUArgs->target=(char *)target; |
5516 | 0 | *pErrorCode=U_ILLEGAL_CHAR_FOUND; |
5517 | 0 | return; |
5518 | 0 | } |
5519 | | |
5520 | | /* get the bytes and the length for the output */ |
5521 | | /* MBCS_OUTPUT_2 */ |
5522 | 0 | value=MBCS_VALUE_2_FROM_STAGE_2(results, stage2Entry, c); |
5523 | | |
5524 | | /* is this code point assigned, or do we use fallbacks? */ |
5525 | 0 | if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || |
5526 | 0 | (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) |
5527 | 0 | ) { |
5528 | 0 | goto unassigned; |
5529 | 0 | } |
5530 | 0 | } |
5531 | 0 | } |
5532 | | |
5533 | | /* write the output character bytes from value and length */ |
5534 | | /* from the first if in the loop we know that targetCapacity>0 */ |
5535 | 0 | if(value<=0xff) { |
5536 | | /* this is easy because we know that there is enough space */ |
5537 | 0 | *target++=(uint8_t)value; |
5538 | 0 | --targetCapacity; |
5539 | 0 | } else /* length==2 */ { |
5540 | 0 | *target++=(uint8_t)(value>>8); |
5541 | 0 | if(2<=targetCapacity) { |
5542 | 0 | *target++=(uint8_t)value; |
5543 | 0 | targetCapacity-=2; |
5544 | 0 | } else { |
5545 | 0 | cnv->charErrorBuffer[0]=(char)value; |
5546 | 0 | cnv->charErrorBufferLength=1; |
5547 | | |
5548 | | /* target overflow */ |
5549 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
5550 | 0 | break; |
5551 | 0 | } |
5552 | 0 | } |
5553 | 0 | continue; |
5554 | | |
5555 | 0 | unassigned: |
5556 | 0 | { |
5557 | | /* |
5558 | | * Try an extension mapping. |
5559 | | * Pass in no source because we don't have UTF-16 input. |
5560 | | * If we have a partial match on c, we will return and revert |
5561 | | * to UTF-8->UTF-16->charset conversion. |
5562 | | */ |
5563 | 0 | static const UChar nul=0; |
5564 | 0 | const UChar *noSource=&nul; |
5565 | 0 | c=_extFromU(cnv, cnv->sharedData, |
5566 | 0 | c, &noSource, noSource, |
5567 | 0 | &target, target+targetCapacity, |
5568 | 0 | NULL, -1, |
5569 | 0 | pFromUArgs->flush, |
5570 | 0 | pErrorCode); |
5571 | |
|
5572 | 0 | if(U_FAILURE(*pErrorCode)) { |
5573 | | /* not mappable or buffer overflow */ |
5574 | 0 | cnv->fromUChar32=c; |
5575 | 0 | break; |
5576 | 0 | } else if(cnv->preFromUFirstCP>=0) { |
5577 | | /* |
5578 | | * Partial match, return and revert to pivoting. |
5579 | | * In normal from-UTF-16 conversion, we would just continue |
5580 | | * but then exit the loop because the extension match would |
5581 | | * have consumed the source. |
5582 | | */ |
5583 | 0 | *pErrorCode=U_USING_DEFAULT_WARNING; |
5584 | 0 | break; |
5585 | 0 | } else { |
5586 | | /* a mapping was written to the target, continue */ |
5587 | | |
5588 | | /* recalculate the targetCapacity after an extension mapping */ |
5589 | 0 | targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); |
5590 | 0 | continue; |
5591 | 0 | } |
5592 | 0 | } |
5593 | 0 | } else { |
5594 | | /* target is full */ |
5595 | 0 | *pErrorCode=U_BUFFER_OVERFLOW_ERROR; |
5596 | 0 | break; |
5597 | 0 | } |
5598 | 0 | } |
5599 | | |
5600 | | /* |
5601 | | * The sourceLimit may have been adjusted before the conversion loop |
5602 | | * to stop before a truncated sequence. |
5603 | | * If so, then collect the truncated sequence now. |
5604 | | */ |
5605 | 0 | if(U_SUCCESS(*pErrorCode) && |
5606 | 0 | cnv->preFromUFirstCP<0 && |
5607 | 0 | source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { |
5608 | 0 | c=utf8->toUBytes[0]=b=*source++; |
5609 | 0 | toULength=1; |
5610 | 0 | toULimit=U8_COUNT_BYTES(b); |
5611 | 0 | while(source<sourceLimit) { |
5612 | 0 | utf8->toUBytes[toULength++]=b=*source++; |
5613 | 0 | c=(c<<6)+b; |
5614 | 0 | } |
5615 | 0 | utf8->toUnicodeStatus=c; |
5616 | 0 | utf8->toULength=toULength; |
5617 | 0 | utf8->mode=toULimit; |
5618 | 0 | } |
5619 | | |
5620 | | /* write back the updated pointers */ |
5621 | 0 | pToUArgs->source=(char *)source; |
5622 | 0 | pFromUArgs->target=(char *)target; |
5623 | 0 | } |
5624 | | |
5625 | | /* miscellaneous ------------------------------------------------------------ */ |
5626 | | |
5627 | | static void U_CALLCONV |
5628 | | ucnv_MBCSGetStarters(const UConverter* cnv, |
5629 | | UBool starters[256], |
5630 | 0 | UErrorCode *) { |
5631 | 0 | const int32_t *state0; |
5632 | 0 | int i; |
5633 | |
|
5634 | 0 | state0=cnv->sharedData->mbcs.stateTable[cnv->sharedData->mbcs.dbcsOnlyState]; |
5635 | 0 | for(i=0; i<256; ++i) { |
5636 | | /* all bytes that cause a state transition from state 0 are lead bytes */ |
5637 | 0 | starters[i]= (UBool)MBCS_ENTRY_IS_TRANSITION(state0[i]); |
5638 | 0 | } |
5639 | 0 | } |
5640 | | |
5641 | | /* |
5642 | | * This is an internal function that allows other converter implementations |
5643 | | * to check whether a byte is a lead byte. |
5644 | | */ |
5645 | | U_CFUNC UBool |
5646 | 0 | ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte) { |
5647 | 0 | return (UBool)MBCS_ENTRY_IS_TRANSITION(sharedData->mbcs.stateTable[0][(uint8_t)byte]); |
5648 | 0 | } |
5649 | | |
5650 | | static void U_CALLCONV |
5651 | | ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs, |
5652 | | int32_t offsetIndex, |
5653 | 0 | UErrorCode *pErrorCode) { |
5654 | 0 | UConverter *cnv=pArgs->converter; |
5655 | 0 | char *p, *subchar; |
5656 | 0 | char buffer[4]; |
5657 | 0 | int32_t length; |
5658 | | |
5659 | | /* first, select between subChar and subChar1 */ |
5660 | 0 | if( cnv->subChar1!=0 && |
5661 | 0 | (cnv->sharedData->mbcs.extIndexes!=NULL ? |
5662 | 0 | cnv->useSubChar1 : |
5663 | 0 | (cnv->invalidUCharBuffer[0]<=0xff)) |
5664 | 0 | ) { |
5665 | | /* select subChar1 if it is set (not 0) and the unmappable Unicode code point is up to U+00ff (IBM MBCS behavior) */ |
5666 | 0 | subchar=(char *)&cnv->subChar1; |
5667 | 0 | length=1; |
5668 | 0 | } else { |
5669 | | /* select subChar in all other cases */ |
5670 | 0 | subchar=(char *)cnv->subChars; |
5671 | 0 | length=cnv->subCharLen; |
5672 | 0 | } |
5673 | | |
5674 | | /* reset the selector for the next code point */ |
5675 | 0 | cnv->useSubChar1=FALSE; |
5676 | |
|
5677 | 0 | if (cnv->sharedData->mbcs.outputType == MBCS_OUTPUT_2_SISO) { |
5678 | 0 | p=buffer; |
5679 | | |
5680 | | /* fromUnicodeStatus contains prevLength */ |
5681 | 0 | switch(length) { |
5682 | 0 | case 1: |
5683 | 0 | if(cnv->fromUnicodeStatus==2) { |
5684 | | /* DBCS mode and SBCS sub char: change to SBCS */ |
5685 | 0 | cnv->fromUnicodeStatus=1; |
5686 | 0 | *p++=UCNV_SI; |
5687 | 0 | } |
5688 | 0 | *p++=subchar[0]; |
5689 | 0 | break; |
5690 | 0 | case 2: |
5691 | 0 | if(cnv->fromUnicodeStatus<=1) { |
5692 | | /* SBCS mode and DBCS sub char: change to DBCS */ |
5693 | 0 | cnv->fromUnicodeStatus=2; |
5694 | 0 | *p++=UCNV_SO; |
5695 | 0 | } |
5696 | 0 | *p++=subchar[0]; |
5697 | 0 | *p++=subchar[1]; |
5698 | 0 | break; |
5699 | 0 | default: |
5700 | 0 | *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
5701 | 0 | return; |
5702 | 0 | } |
5703 | 0 | subchar=buffer; |
5704 | 0 | length=(int32_t)(p-buffer); |
5705 | 0 | } |
5706 | | |
5707 | 0 | ucnv_cbFromUWriteBytes(pArgs, subchar, length, offsetIndex, pErrorCode); |
5708 | 0 | } |
5709 | | |
5710 | | U_CFUNC UConverterType |
5711 | 0 | ucnv_MBCSGetType(const UConverter* converter) { |
5712 | | /* SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but here we cheat a little */ |
5713 | 0 | if(converter->sharedData->mbcs.countStates==1) { |
5714 | 0 | return (UConverterType)UCNV_SBCS; |
5715 | 0 | } else if((converter->sharedData->mbcs.outputType&0xff)==MBCS_OUTPUT_2_SISO) { |
5716 | 0 | return (UConverterType)UCNV_EBCDIC_STATEFUL; |
5717 | 0 | } else if(converter->sharedData->staticData->minBytesPerChar==2 && converter->sharedData->staticData->maxBytesPerChar==2) { |
5718 | 0 | return (UConverterType)UCNV_DBCS; |
5719 | 0 | } |
5720 | 0 | return (UConverterType)UCNV_MBCS; |
5721 | 0 | } |
5722 | | |
5723 | | #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ |