Coverage Report

Created: 2024-04-24 06:23

/src/icu/source/common/ucnvmbcs.cpp
Line
Count
Source (jump to first uncovered line)
1
// © 2016 and later: Unicode, Inc. and others.
2
// License & terms of use: http://www.unicode.org/copyright.html
3
/*
4
******************************************************************************
5
*
6
*   Copyright (C) 2000-2016, International Business Machines
7
*   Corporation and others.  All Rights Reserved.
8
*
9
******************************************************************************
10
*   file name:  ucnvmbcs.cpp
11
*   encoding:   UTF-8
12
*   tab size:   8 (not used)
13
*   indentation:4
14
*
15
*   created on: 2000jul03
16
*   created by: Markus W. Scherer
17
*
18
*   The current code in this file replaces the previous implementation
19
*   of conversion code from multi-byte codepages to Unicode and back.
20
*   This implementation supports the following:
21
*   - legacy variable-length codepages with up to 4 bytes per character
22
*   - all Unicode code points (up to 0x10ffff)
23
*   - efficient distinction of unassigned vs. illegal byte sequences
24
*   - it is possible in fromUnicode() to directly deal with simple
25
*     stateful encodings (used for EBCDIC_STATEFUL)
26
*   - it is possible to convert Unicode code points
27
*     to a single zero byte (but not as a fallback except for SBCS)
28
*
29
*   Remaining limitations in fromUnicode:
30
*   - byte sequences must not have leading zero bytes
31
*   - except for SBCS codepages: no fallback mapping from Unicode to a zero byte
32
*   - limitation to up to 4 bytes per character
33
*
34
*   ICU 2.8 (late 2003) adds a secondary data structure which lifts some of these
35
*   limitations and adds m:n character mappings and other features.
36
*   See ucnv_ext.h for details.
37
*
38
*   Change history: 
39
*
40
*    5/6/2001       Ram       Moved  MBCS_SINGLE_RESULT_FROM_U,MBCS_STAGE_2_FROM_U,
41
*                             MBCS_VALUE_2_FROM_STAGE_2, MBCS_VALUE_4_FROM_STAGE_2
42
*                             macros to ucnvmbcs.h file
43
*/
44
45
#include "unicode/utypes.h"
46
47
#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
48
49
#include "unicode/ucnv.h"
50
#include "unicode/ucnv_cb.h"
51
#include "unicode/udata.h"
52
#include "unicode/uset.h"
53
#include "unicode/utf8.h"
54
#include "unicode/utf16.h"
55
#include "ucnv_bld.h"
56
#include "ucnvmbcs.h"
57
#include "ucnv_ext.h"
58
#include "ucnv_cnv.h"
59
#include "cmemory.h"
60
#include "cstring.h"
61
#include "umutex.h"
62
#include "ustr_imp.h"
63
64
/* control optimizations according to the platform */
65
#define MBCS_UNROLL_SINGLE_TO_BMP 1
66
#define MBCS_UNROLL_SINGLE_FROM_BMP 0
67
68
/*
69
 * _MBCSHeader versions 5.3 & 4.3
70
 * (Note that the _MBCSHeader version is in addition to the converter formatVersion.)
71
 *
72
 * This version is optional. Version 5 is used for incompatible data format changes.
73
 * makeconv will continue to generate version 4 files if possible.
74
 *
75
 * Changes from version 4:
76
 *
77
 * The main difference is an additional _MBCSHeader field with
78
 * - the length (number of uint32_t) of the _MBCSHeader
79
 * - flags for further incompatible data format changes
80
 * - flags for further, backward compatible data format changes
81
 *
82
 * The MBCS_OPT_FROM_U flag indicates that most of the fromUnicode data is omitted from
83
 * the file and needs to be reconstituted at load time.
84
 * This requires a utf8Friendly format with an additional mbcsIndex table for fast
85
 * (and UTF-8-friendly) fromUnicode conversion for Unicode code points up to maxFastUChar.
86
 * (For details about these structures see below, and see ucnvmbcs.h.)
87
 *
88
 *   utf8Friendly also implies that the fromUnicode mappings are stored in ascending order
89
 *   of the Unicode code points. (This requires that the .ucm file has the |0 etc.
90
 *   precision markers for all mappings.)
91
 *
92
 *   All fallbacks have been moved to the extension table, leaving only roundtrips in the
93
 *   omitted data that can be reconstituted from the toUnicode data.
94
 *
95
 *   Of the stage 2 table, the part corresponding to maxFastUChar and below is omitted.
96
 *   With only roundtrip mappings in the base fromUnicode data, this part is fully
97
 *   redundant with the mbcsIndex and will be reconstituted from that (also using the
98
 *   stage 1 table which contains the information about how stage 2 was compacted).
99
 *
100
 *   The rest of the stage 2 table, the part for code points above maxFastUChar,
101
 *   is stored in the file and will be appended to the reconstituted part.
102
 *
103
 *   The entire fromUBytes array is omitted from the file and will be reconstitued.
104
 *   This is done by enumerating all toUnicode roundtrip mappings, performing
105
 *   each mapping (using the stage 1 and reconstituted stage 2 tables) and
106
 *   writing instead of reading the byte values.
107
 *
108
 * _MBCSHeader version 4.3
109
 *
110
 * Change from version 4.2:
111
 * - Optional utf8Friendly data structures, with 64-entry stage 3 block
112
 *   allocation for parts of the BMP, and an additional mbcsIndex in non-SBCS
113
 *   files which can be used instead of stages 1 & 2.
114
 *   Faster lookups for roundtrips from most commonly used characters,
115
 *   and lookups from UTF-8 byte sequences with a natural bit distribution.
116
 *   See ucnvmbcs.h for more details.
117
 *
118
 * Change from version 4.1:
119
 * - Added an optional extension table structure at the end of the .cnv file.
120
 *   It is present if the upper bits of the header flags field contains a non-zero
121
 *   byte offset to it.
122
 *   Files that contain only a conversion table and no base table
123
 *   use the special outputType MBCS_OUTPUT_EXT_ONLY.
124
 *   These contain the base table name between the MBCS header and the extension
125
 *   data.
126
 *
127
 * Change from version 4.0:
128
 * - Replace header.reserved with header.fromUBytesLength so that all
129
 *   fields in the data have length.
130
 *
131
 * Changes from version 3 (for performance improvements):
132
 * - new bit distribution for state table entries
133
 * - reordered action codes
134
 * - new data structure for single-byte fromUnicode
135
 *   + stage 2 only contains indexes
136
 *   + stage 3 stores 16 bits per character with classification bits 15..8
137
 * - no multiplier for stage 1 entries
138
 * - stage 2 for non-single-byte codepages contains the index and the flags in
139
 *   one 32-bit value
140
 * - 2-byte and 4-byte fromUnicode results are stored directly as 16/32-bit integers
141
 *
142
 * For more details about old versions of the MBCS data structure, see
143
 * the corresponding versions of this file.
144
 *
145
 * Converting stateless codepage data ---------------------------------------***
146
 * (or codepage data with simple states) to Unicode.
147
 *
148
 * Data structure and algorithm for converting from complex legacy codepages
149
 * to Unicode. (Designed before 2000-may-22.)
150
 *
151
 * The basic idea is that the structure of legacy codepages can be described
152
 * with state tables.
153
 * When reading a byte stream, each input byte causes a state transition.
154
 * Some transitions result in the output of a code point, some result in
155
 * "unassigned" or "illegal" output.
156
 * This is used here for character conversion.
157
 *
158
 * The data structure begins with a state table consisting of a row
159
 * per state, with 256 entries (columns) per row for each possible input
160
 * byte value.
161
 * Each entry is 32 bits wide, with two formats distinguished by
162
 * the sign bit (bit 31):
163
 *
164
 * One format for transitional entries (bit 31 not set) for non-final bytes, and
165
 * one format for final entries (bit 31 set).
166
 * Both formats contain the number of the next state in the same bit
167
 * positions.
168
 * State 0 is the initial state.
169
 *
170
 * Most of the time, the offset values of subsequent states are added
171
 * up to a scalar value. This value will eventually be the index of
172
 * the Unicode code point in a table that follows the state table.
173
 * The effect is that the code points for final state table rows
174
 * are contiguous. The code points of final state rows follow each other
175
 * in the order of the references to those final states by previous
176
 * states, etc.
177
 *
178
 * For some terminal states, the offset is itself the output Unicode
179
 * code point (16 bits for a BMP code point or 20 bits for a supplementary
180
 * code point (stored as code point minus 0x10000 so that 20 bits are enough).
181
 * For others, the code point in the Unicode table is stored with either
182
 * one or two code units: one for BMP code points, two for a pair of
183
 * surrogates.
184
 * All code points for a final state entry take up the same number of code
185
 * units, regardless of whether they all actually _use_ the same number
186
 * of code units. This is necessary for simple array access.
187
 *
188
 * An additional feature comes in with what in ICU is called "fallback"
189
 * mappings:
190
 *
191
 * In addition to round-trippable, precise, 1:1 mappings, there are often
192
 * mappings defined between similar, though not the same, characters.
193
 * Typically, such mappings occur only in fromUnicode mapping tables because
194
 * Unicode has a superset repertoire of most other codepages. However, it
195
 * is possible to provide such mappings in the toUnicode tables, too.
196
 * In this case, the fallback mappings are partly integrated into the
197
 * general state tables because the structure of the encoding includes their
198
 * byte sequences.
199
 * For final entries in an initial state, fallback mappings are stored in
200
 * the entry itself like with roundtrip mappings.
201
 * For other final entries, they are stored in the code units table if
202
 * the entry is for a pair of code units.
203
 * For single-unit results in the code units table, there is no space to
204
 * alternatively hold a fallback mapping; in this case, the code unit
205
 * is stored as U+fffe (unassigned), and the fallback mapping needs to
206
 * be looked up by the scalar offset value in a separate table.
207
 *
208
 * "Unassigned" state entries really mean "structurally unassigned",
209
 * i.e., such a byte sequence will never have a mapping result.
210
 *
211
 * The interpretation of the bits in each entry is as follows:
212
 *
213
 * Bit 31 not set, not a terminal entry ("transitional"):
214
 * 30..24 next state
215
 * 23..0  offset delta, to be added up
216
 *
217
 * Bit 31 set, terminal ("final") entry:
218
 * 30..24 next state (regardless of action code)
219
 * 23..20 action code:
220
 *        action codes 0 and 1 result in precise-mapping Unicode code points
221
 *        0  valid byte sequence
222
 *           19..16 not used, 0
223
 *           15..0  16-bit Unicode BMP code point
224
 *                  never U+fffe or U+ffff
225
 *        1  valid byte sequence
226
 *           19..0  20-bit Unicode supplementary code point
227
 *                  never U+fffe or U+ffff
228
 *
229
 *        action codes 2 and 3 result in fallback (unidirectional-mapping) Unicode code points
230
 *        2  valid byte sequence (fallback)
231
 *           19..16 not used, 0
232
 *           15..0  16-bit Unicode BMP code point as fallback result
233
 *        3  valid byte sequence (fallback)
234
 *           19..0  20-bit Unicode supplementary code point as fallback result
235
 *
236
 *        action codes 4 and 5 may result in roundtrip/fallback/unassigned/illegal results
237
 *        depending on the code units they result in
238
 *        4  valid byte sequence
239
 *           19..9  not used, 0
240
 *            8..0  final offset delta
241
 *                  pointing to one 16-bit code unit which may be
242
 *                  fffe  unassigned -- look for a fallback for this offset
243
 *                  ffff  illegal
244
 *        5  valid byte sequence
245
 *           19..9  not used, 0
246
 *            8..0  final offset delta
247
 *                  pointing to two 16-bit code units
248
 *                  (typically UTF-16 surrogates)
249
 *                  the result depends on the first code unit as follows:
250
 *                  0000..d7ff  roundtrip BMP code point (1st alone)
251
 *                  d800..dbff  roundtrip surrogate pair (1st, 2nd)
252
 *                  dc00..dfff  fallback surrogate pair (1st-400, 2nd)
253
 *                  e000        roundtrip BMP code point (2nd alone)
254
 *                  e001        fallback BMP code point (2nd alone)
255
 *                  fffe        unassigned
256
 *                  ffff        illegal
257
 *           (the final offset deltas are at most 255 * 2,
258
 *            times 2 because of storing code unit pairs)
259
 *
260
 *        6  unassigned byte sequence
261
 *           19..16 not used, 0
262
 *           15..0  16-bit Unicode BMP code point U+fffe (new with version 2)
263
 *                  this does not contain a final offset delta because the main
264
 *                  purpose of this action code is to save scalar offset values;
265
 *                  therefore, fallback values cannot be assigned to byte
266
 *                  sequences that result in this action code
267
 *        7  illegal byte sequence
268
 *           19..16 not used, 0
269
 *           15..0  16-bit Unicode BMP code point U+ffff (new with version 2)
270
 *        8  state change only
271
 *           19..0  not used, 0
272
 *           useful for state changes in simple stateful encodings,
273
 *           at Shift-In/Shift-Out codes
274
 *
275
 *
276
 *        9..15 reserved for future use
277
 *           current implementations will only perform a state change
278
 *           and ignore bits 19..0
279
 *
280
 * An encoding with contiguous ranges of unassigned byte sequences, like
281
 * Shift-JIS and especially EUC-TW, can be stored efficiently by having
282
 * at least two states for the trail bytes:
283
 * One trail byte state that results in code points, and one that only
284
 * has "unassigned" and "illegal" terminal states.
285
 *
286
 * Note: partly by accident, this data structure supports simple stateful
287
 * encodings without any additional logic.
288
 * Currently, only simple Shift-In/Shift-Out schemes are handled with
289
 * appropriate state tables (especially EBCDIC_STATEFUL!).
290
 *
291
 * MBCS version 2 added:
292
 * unassigned and illegal action codes have U+fffe and U+ffff
293
 * instead of unused bits; this is useful for _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP()
294
 *
295
 * Converting from Unicode to codepage bytes --------------------------------***
296
 *
297
 * The conversion data structure for fromUnicode is designed for the known
298
 * structure of Unicode. It maps from 21-bit code points (0..0x10ffff) to
299
 * a sequence of 1..4 bytes, in addition to a flag that indicates if there is
300
 * a roundtrip mapping.
301
 *
302
 * The lookup is done with a 3-stage trie, using 11/6/4 bits for stage 1/2/3
303
 * like in the character properties table.
304
 * The beginning of the trie is at offsetFromUTable, the beginning of stage 3
305
 * with the resulting bytes is at offsetFromUBytes.
306
 *
307
 * Beginning with version 4, single-byte codepages have a significantly different
308
 * trie compared to other codepages.
309
 * In all cases, the entry in stage 1 is directly the index of the block of
310
 * 64 entries in stage 2.
311
 *
312
 * Single-byte lookup:
313
 *
314
 * Stage 2 only contains 16-bit indexes directly to the 16-blocks in stage 3.
315
 * Stage 3 contains one 16-bit word per result:
316
 * Bits 15..8 indicate the kind of result:
317
 *    f  roundtrip result
318
 *    c  fallback result from private-use code point
319
 *    8  fallback result from other code points
320
 *    0  unassigned
321
 * Bits 7..0 contain the codepage byte. A zero byte is always possible.
322
 *
323
 * In version 4.3, the runtime code can build an sbcsIndex for a utf8Friendly
324
 * file. For 2-byte UTF-8 byte sequences and some 3-byte sequences the lookup
325
 * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3.
326
 * ASCII code points can be looked up with a linear array access into stage 3.
327
 * See maxFastUChar and other details in ucnvmbcs.h.
328
 *
329
 * Multi-byte lookup:
330
 *
331
 * Stage 2 contains a 32-bit word for each 16-block in stage 3:
332
 * Bits 31..16 contain flags for which stage 3 entries contain roundtrip results
333
 *             test: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)
334
 *             If this test is false, then a non-zero result will be interpreted as
335
 *             a fallback mapping.
336
 * Bits 15..0  contain the index to stage 3, which must be multiplied by 16*(bytes per char)
337
 *
338
 * Stage 3 contains 2, 3, or 4 bytes per result.
339
 * 2 or 4 bytes are stored as uint16_t/uint32_t in platform endianness,
340
 * while 3 bytes are stored as bytes in big-endian order.
341
 * Leading zero bytes are ignored, and the number of bytes is counted.
342
 * A zero byte mapping result is possible as a roundtrip result.
343
 * For some output types, the actual result is processed from this;
344
 * see ucnv_MBCSFromUnicodeWithOffsets().
345
 *
346
 * Note that stage 1 always contains 0x440=1088 entries (0x440==0x110000>>10),
347
 * or (version 3 and up) for BMP-only codepages, it contains 64 entries.
348
 *
349
 * In version 4.3, a utf8Friendly file contains an mbcsIndex table.
350
 * For 2-byte UTF-8 byte sequences and most 3-byte sequences the lookup
351
 * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3.
352
 * ASCII code points can be looked up with a linear array access into stage 3.
353
 * See maxFastUChar, mbcsIndex and other details in ucnvmbcs.h.
354
 *
355
 * In version 3, stage 2 blocks may overlap by multiples of the multiplier
356
 * for compaction.
357
 * In version 4, stage 2 blocks (and for single-byte codepages, stage 3 blocks)
358
 * may overlap by any number of entries.
359
 *
360
 * MBCS version 2 added:
361
 * the converter checks for known output types, which allows
362
 * adding new ones without crashing an unaware converter
363
 */
364
365
/**
366
 * Callback from ucnv_MBCSEnumToUnicode(), takes 32 mappings from
367
 * consecutive sequences of bytes, starting from the one encoded in value,
368
 * to Unicode code points. (Multiple mappings to reduce per-function call overhead.)
369
 * Does not currently support m:n mappings or reverse fallbacks.
370
 * This function will not be called for sequences of bytes with leading zeros.
371
 *
372
 * @param context an opaque pointer, as passed into ucnv_MBCSEnumToUnicode()
373
 * @param value contains 1..4 bytes of the first byte sequence, right-aligned
374
 * @param codePoints resulting Unicode code points, or negative if a byte sequence does
375
 *        not map to anything
376
 * @return TRUE to continue enumeration, FALSE to stop
377
 */
378
typedef UBool U_CALLCONV
379
UConverterEnumToUCallback(const void *context, uint32_t value, UChar32 codePoints[32]);
380
381
static void U_CALLCONV
382
ucnv_MBCSLoad(UConverterSharedData *sharedData,
383
          UConverterLoadArgs *pArgs,
384
          const uint8_t *raw,
385
          UErrorCode *pErrorCode);
386
387
static void U_CALLCONV
388
ucnv_MBCSUnload(UConverterSharedData *sharedData);
389
390
static void U_CALLCONV
391
ucnv_MBCSOpen(UConverter *cnv,
392
              UConverterLoadArgs *pArgs,
393
              UErrorCode *pErrorCode);
394
395
static UChar32 U_CALLCONV
396
ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs,
397
                  UErrorCode *pErrorCode);
398
399
static void U_CALLCONV
400
ucnv_MBCSGetStarters(const UConverter* cnv,
401
                 UBool starters[256],
402
                 UErrorCode *pErrorCode);
403
404
U_CDECL_BEGIN
405
static const char* U_CALLCONV
406
ucnv_MBCSGetName(const UConverter *cnv);
407
U_CDECL_END
408
409
static void U_CALLCONV
410
ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs,
411
              int32_t offsetIndex,
412
              UErrorCode *pErrorCode);
413
414
static UChar32 U_CALLCONV
415
ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs,
416
                  UErrorCode *pErrorCode);
417
418
static void U_CALLCONV
419
ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
420
                  UConverterToUnicodeArgs *pToUArgs,
421
                  UErrorCode *pErrorCode);
422
423
static void U_CALLCONV
424
ucnv_MBCSGetUnicodeSet(const UConverter *cnv,
425
                   const USetAdder *sa,
426
                   UConverterUnicodeSet which,
427
                   UErrorCode *pErrorCode);
428
429
static void U_CALLCONV
430
ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
431
                  UConverterToUnicodeArgs *pToUArgs,
432
                  UErrorCode *pErrorCode);
433
434
static const UConverterImpl _SBCSUTF8Impl={
435
    UCNV_MBCS,
436
437
    ucnv_MBCSLoad,
438
    ucnv_MBCSUnload,
439
440
    ucnv_MBCSOpen,
441
    NULL,
442
    NULL,
443
444
    ucnv_MBCSToUnicodeWithOffsets,
445
    ucnv_MBCSToUnicodeWithOffsets,
446
    ucnv_MBCSFromUnicodeWithOffsets,
447
    ucnv_MBCSFromUnicodeWithOffsets,
448
    ucnv_MBCSGetNextUChar,
449
450
    ucnv_MBCSGetStarters,
451
    ucnv_MBCSGetName,
452
    ucnv_MBCSWriteSub,
453
    NULL,
454
    ucnv_MBCSGetUnicodeSet,
455
456
    NULL,
457
    ucnv_SBCSFromUTF8
458
};
459
460
static const UConverterImpl _DBCSUTF8Impl={
461
    UCNV_MBCS,
462
463
    ucnv_MBCSLoad,
464
    ucnv_MBCSUnload,
465
466
    ucnv_MBCSOpen,
467
    NULL,
468
    NULL,
469
470
    ucnv_MBCSToUnicodeWithOffsets,
471
    ucnv_MBCSToUnicodeWithOffsets,
472
    ucnv_MBCSFromUnicodeWithOffsets,
473
    ucnv_MBCSFromUnicodeWithOffsets,
474
    ucnv_MBCSGetNextUChar,
475
476
    ucnv_MBCSGetStarters,
477
    ucnv_MBCSGetName,
478
    ucnv_MBCSWriteSub,
479
    NULL,
480
    ucnv_MBCSGetUnicodeSet,
481
482
    NULL,
483
    ucnv_DBCSFromUTF8
484
};
485
486
static const UConverterImpl _MBCSImpl={
487
    UCNV_MBCS,
488
489
    ucnv_MBCSLoad,
490
    ucnv_MBCSUnload,
491
492
    ucnv_MBCSOpen,
493
    NULL,
494
    NULL,
495
496
    ucnv_MBCSToUnicodeWithOffsets,
497
    ucnv_MBCSToUnicodeWithOffsets,
498
    ucnv_MBCSFromUnicodeWithOffsets,
499
    ucnv_MBCSFromUnicodeWithOffsets,
500
    ucnv_MBCSGetNextUChar,
501
502
    ucnv_MBCSGetStarters,
503
    ucnv_MBCSGetName,
504
    ucnv_MBCSWriteSub,
505
    NULL,
506
    ucnv_MBCSGetUnicodeSet,
507
    NULL,
508
    NULL
509
};
510
511
/* Static data is in tools/makeconv/ucnvstat.c for data-based
512
 * converters. Be sure to update it as well.
513
 */
514
515
const UConverterSharedData _MBCSData={
516
    sizeof(UConverterSharedData), 1,
517
    NULL, NULL, FALSE, TRUE, &_MBCSImpl,
518
    0, UCNV_MBCS_TABLE_INITIALIZER
519
};
520
521
522
/* GB 18030 data ------------------------------------------------------------ */
523
524
/* helper macros for linear values for GB 18030 four-byte sequences */
525
0
#define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d))
526
527
0
#define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30)
528
529
#define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff)
530
531
/*
532
 * Some ranges of GB 18030 where both the Unicode code points and the
533
 * GB four-byte sequences are contiguous and are handled algorithmically by
534
 * the special callback functions below.
535
 * The values are start & end of Unicode & GB codes.
536
 *
537
 * Note that single surrogates are not mapped by GB 18030
538
 * as of the re-released mapping tables from 2000-nov-30.
539
 */
540
static const uint32_t
541
gb18030Ranges[14][4]={
542
    {0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35)},
543
    {0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738)},
544
    {0x0452, 0x1E3E, LINEAR(0x8130D330), LINEAR(0x8135F436)},
545
    {0x1E40, 0x200F, LINEAR(0x8135F438), LINEAR(0x8136A531)},
546
    {0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534)},
547
    {0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38)},
548
    {0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537)},
549
    {0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32)},
550
    {0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237)},
551
    {0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733)},
552
    {0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837)},
553
    {0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638)},
554
    {0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931)},
555
    {0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439)}
556
};
557
558
/* bit flag for UConverter.options indicating GB 18030 special handling */
559
0
#define _MBCS_OPTION_GB18030 0x8000
560
561
/* bit flag for UConverter.options indicating KEIS,JEF,JIF special handling */
562
0
#define _MBCS_OPTION_KEIS 0x01000
563
0
#define _MBCS_OPTION_JEF  0x02000
564
0
#define _MBCS_OPTION_JIPS 0x04000
565
566
0
#define KEIS_SO_CHAR_1 0x0A
567
0
#define KEIS_SO_CHAR_2 0x42
568
0
#define KEIS_SI_CHAR_1 0x0A
569
0
#define KEIS_SI_CHAR_2 0x41
570
571
0
#define JEF_SO_CHAR 0x28
572
0
#define JEF_SI_CHAR 0x29
573
574
0
#define JIPS_SO_CHAR_1 0x1A
575
0
#define JIPS_SO_CHAR_2 0x70
576
0
#define JIPS_SI_CHAR_1 0x1A
577
0
#define JIPS_SI_CHAR_2 0x71
578
579
enum SISO_Option {
580
    SI,
581
    SO
582
};
583
typedef enum SISO_Option SISO_Option;
584
585
0
static int32_t getSISOBytes(SISO_Option option, uint32_t cnvOption, uint8_t *value) {
586
0
    int32_t SISOLength = 0;
587
588
0
    switch (option) {
589
0
        case SI:
590
0
            if ((cnvOption&_MBCS_OPTION_KEIS)!=0) {
591
0
                value[0] = KEIS_SI_CHAR_1;
592
0
                value[1] = KEIS_SI_CHAR_2;
593
0
                SISOLength = 2;
594
0
            } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) {
595
0
                value[0] = JEF_SI_CHAR;
596
0
                SISOLength = 1;
597
0
            } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) {
598
0
                value[0] = JIPS_SI_CHAR_1;
599
0
                value[1] = JIPS_SI_CHAR_2;
600
0
                SISOLength = 2;
601
0
            } else {
602
0
                value[0] = UCNV_SI;
603
0
                SISOLength = 1;
604
0
            }
605
0
            break;
606
0
        case SO:
607
0
            if ((cnvOption&_MBCS_OPTION_KEIS)!=0) {
608
0
                value[0] = KEIS_SO_CHAR_1;
609
0
                value[1] = KEIS_SO_CHAR_2;
610
0
                SISOLength = 2;
611
0
            } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) {
612
0
                value[0] = JEF_SO_CHAR;
613
0
                SISOLength = 1;
614
0
            } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) {
615
0
                value[0] = JIPS_SO_CHAR_1;
616
0
                value[1] = JIPS_SO_CHAR_2;
617
0
                SISOLength = 2;
618
0
            } else {
619
0
                value[0] = UCNV_SO;
620
0
                SISOLength = 1;
621
0
            }
622
0
            break;
623
0
        default:
624
            /* Should never happen. */
625
0
            break;
626
0
    }
627
628
0
    return SISOLength;
629
0
}
630
631
/* Miscellaneous ------------------------------------------------------------ */
632
633
/* similar to ucnv_MBCSGetNextUChar() but recursive */
634
static UBool
635
enumToU(UConverterMBCSTable *mbcsTable, int8_t stateProps[],
636
        int32_t state, uint32_t offset,
637
        uint32_t value,
638
        UConverterEnumToUCallback *callback, const void *context,
639
0
        UErrorCode *pErrorCode) {
640
0
    UChar32 codePoints[32];
641
0
    const int32_t *row;
642
0
    const uint16_t *unicodeCodeUnits;
643
0
    UChar32 anyCodePoints;
644
0
    int32_t b, limit;
645
646
0
    row=mbcsTable->stateTable[state];
647
0
    unicodeCodeUnits=mbcsTable->unicodeCodeUnits;
648
649
0
    value<<=8;
650
0
    anyCodePoints=-1;  /* becomes non-negative if there is a mapping */
651
652
0
    b=(stateProps[state]&0x38)<<2;
653
0
    if(b==0 && stateProps[state]>=0x40) {
654
        /* skip byte sequences with leading zeros because they are not stored in the fromUnicode table */
655
0
        codePoints[0]=U_SENTINEL;
656
0
        b=1;
657
0
    }
658
0
    limit=((stateProps[state]&7)+1)<<5;
659
0
    while(b<limit) {
660
0
        int32_t entry=row[b];
661
0
        if(MBCS_ENTRY_IS_TRANSITION(entry)) {
662
0
            int32_t nextState=MBCS_ENTRY_TRANSITION_STATE(entry);
663
0
            if(stateProps[nextState]>=0) {
664
                /* recurse to a state with non-ignorable actions */
665
0
                if(!enumToU(
666
0
                        mbcsTable, stateProps, nextState,
667
0
                        offset+MBCS_ENTRY_TRANSITION_OFFSET(entry),
668
0
                        value|(uint32_t)b,
669
0
                        callback, context,
670
0
                        pErrorCode)) {
671
0
                    return FALSE;
672
0
                }
673
0
            }
674
0
            codePoints[b&0x1f]=U_SENTINEL;
675
0
        } else {
676
0
            UChar32 c;
677
0
            int32_t action;
678
679
            /*
680
             * An if-else-if chain provides more reliable performance for
681
             * the most common cases compared to a switch.
682
             */
683
0
            action=MBCS_ENTRY_FINAL_ACTION(entry);
684
0
            if(action==MBCS_STATE_VALID_DIRECT_16) {
685
                /* output BMP code point */
686
0
                c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
687
0
            } else if(action==MBCS_STATE_VALID_16) {
688
0
                int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry);
689
0
                c=unicodeCodeUnits[finalOffset];
690
0
                if(c<0xfffe) {
691
                    /* output BMP code point */
692
0
                } else {
693
0
                    c=U_SENTINEL;
694
0
                }
695
0
            } else if(action==MBCS_STATE_VALID_16_PAIR) {
696
0
                int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry);
697
0
                c=unicodeCodeUnits[finalOffset++];
698
0
                if(c<0xd800) {
699
                    /* output BMP code point below 0xd800 */
700
0
                } else if(c<=0xdbff) {
701
                    /* output roundtrip or fallback supplementary code point */
702
0
                    c=((c&0x3ff)<<10)+unicodeCodeUnits[finalOffset]+(0x10000-0xdc00);
703
0
                } else if(c==0xe000) {
704
                    /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
705
0
                    c=unicodeCodeUnits[finalOffset];
706
0
                } else {
707
0
                    c=U_SENTINEL;
708
0
                }
709
0
            } else if(action==MBCS_STATE_VALID_DIRECT_20) {
710
                /* output supplementary code point */
711
0
                c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000);
712
0
            } else {
713
0
                c=U_SENTINEL;
714
0
            }
715
716
0
            codePoints[b&0x1f]=c;
717
0
            anyCodePoints&=c;
718
0
        }
719
0
        if(((++b)&0x1f)==0) {
720
0
            if(anyCodePoints>=0) {
721
0
                if(!callback(context, value|(uint32_t)(b-0x20), codePoints)) {
722
0
                    return FALSE;
723
0
                }
724
0
                anyCodePoints=-1;
725
0
            }
726
0
        }
727
0
    }
728
0
    return TRUE;
729
0
}
730
731
/*
732
 * Only called if stateProps[state]==-1.
733
 * A recursive call may do stateProps[state]|=0x40 if this state is the target of an
734
 * MBCS_STATE_CHANGE_ONLY.
735
 */
736
static int8_t
737
0
getStateProp(const int32_t (*stateTable)[256], int8_t stateProps[], int state) {
738
0
    const int32_t *row;
739
0
    int32_t min, max, entry, nextState;
740
741
0
    row=stateTable[state];
742
0
    stateProps[state]=0;
743
744
    /* find first non-ignorable state */
745
0
    for(min=0;; ++min) {
746
0
        entry=row[min];
747
0
        nextState=MBCS_ENTRY_STATE(entry);
748
0
        if(stateProps[nextState]==-1) {
749
0
            getStateProp(stateTable, stateProps, nextState);
750
0
        }
751
0
        if(MBCS_ENTRY_IS_TRANSITION(entry)) {
752
0
            if(stateProps[nextState]>=0) {
753
0
                break;
754
0
            }
755
0
        } else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) {
756
0
            break;
757
0
        }
758
0
        if(min==0xff) {
759
0
            stateProps[state]=-0x40;  /* (int8_t)0xc0 */
760
0
            return stateProps[state];
761
0
        }
762
0
    }
763
0
    stateProps[state]|=(int8_t)((min>>5)<<3);
764
765
    /* find last non-ignorable state */
766
0
    for(max=0xff; min<max; --max) {
767
0
        entry=row[max];
768
0
        nextState=MBCS_ENTRY_STATE(entry);
769
0
        if(stateProps[nextState]==-1) {
770
0
            getStateProp(stateTable, stateProps, nextState);
771
0
        }
772
0
        if(MBCS_ENTRY_IS_TRANSITION(entry)) {
773
0
            if(stateProps[nextState]>=0) {
774
0
                break;
775
0
            }
776
0
        } else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) {
777
0
            break;
778
0
        }
779
0
    }
780
0
    stateProps[state]|=(int8_t)(max>>5);
781
782
    /* recurse further and collect direct-state information */
783
0
    while(min<=max) {
784
0
        entry=row[min];
785
0
        nextState=MBCS_ENTRY_STATE(entry);
786
0
        if(stateProps[nextState]==-1) {
787
0
            getStateProp(stateTable, stateProps, nextState);
788
0
        }
789
0
        if(MBCS_ENTRY_IS_FINAL(entry)) {
790
0
            stateProps[nextState]|=0x40;
791
0
            if(MBCS_ENTRY_FINAL_ACTION(entry)<=MBCS_STATE_FALLBACK_DIRECT_20) {
792
0
                stateProps[state]|=0x40;
793
0
            }
794
0
        }
795
0
        ++min;
796
0
    }
797
0
    return stateProps[state];
798
0
}
799
800
/*
801
 * Internal function enumerating the toUnicode data of an MBCS converter.
802
 * Currently only used for reconstituting data for a MBCS_OPT_NO_FROM_U
803
 * table, but could also be used for a future ucnv_getUnicodeSet() option
804
 * that includes reverse fallbacks (after updating this function's implementation).
805
 * Currently only handles roundtrip mappings.
806
 * Does not currently handle extensions.
807
 */
808
static void
809
ucnv_MBCSEnumToUnicode(UConverterMBCSTable *mbcsTable,
810
                       UConverterEnumToUCallback *callback, const void *context,
811
0
                       UErrorCode *pErrorCode) {
812
    /*
813
     * Properties for each state, to speed up the enumeration.
814
     * Ignorable actions are unassigned/illegal/state-change-only:
815
     * They do not lead to mappings.
816
     *
817
     * Bits 7..6:
818
     * 1 direct/initial state (stateful converters have multiple)
819
     * 0 non-initial state with transitions or with non-ignorable result actions
820
     * -1 final state with only ignorable actions
821
     *
822
     * Bits 5..3:
823
     * The lowest byte value with non-ignorable actions is
824
     * value<<5 (rounded down).
825
     *
826
     * Bits 2..0:
827
     * The highest byte value with non-ignorable actions is
828
     * (value<<5)&0x1f (rounded up).
829
     */
830
0
    int8_t stateProps[MBCS_MAX_STATE_COUNT];
831
0
    int32_t state;
832
833
0
    uprv_memset(stateProps, -1, sizeof(stateProps));
834
835
    /* recurse from state 0 and set all stateProps */
836
0
    getStateProp(mbcsTable->stateTable, stateProps, 0);
837
838
0
    for(state=0; state<mbcsTable->countStates; ++state) {
839
        /*if(stateProps[state]==-1) {
840
            printf("unused/unreachable <icu:state> %d\n", state);
841
        }*/
842
0
        if(stateProps[state]>=0x40) {
843
            /* start from each direct state */
844
0
            enumToU(
845
0
                mbcsTable, stateProps, state, 0, 0,
846
0
                callback, context,
847
0
                pErrorCode);
848
0
        }
849
0
    }
850
0
}
851
852
U_CFUNC void 
853
ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData,
854
                                         const USetAdder *sa,
855
                                         UConverterUnicodeSet which,
856
                                         UConverterSetFilter filter,
857
0
                                         UErrorCode *pErrorCode) {
858
0
    const UConverterMBCSTable *mbcsTable;
859
0
    const uint16_t *table;
860
861
0
    uint32_t st3;
862
0
    uint16_t st1, maxStage1, st2;
863
864
0
    UChar32 c;
865
866
    /* enumerate the from-Unicode trie table */
867
0
    mbcsTable=&sharedData->mbcs;
868
0
    table=mbcsTable->fromUnicodeTable;
869
0
    if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
870
0
        maxStage1=0x440;
871
0
    } else {
872
0
        maxStage1=0x40;
873
0
    }
874
875
0
    c=0; /* keep track of the current code point while enumerating */
876
877
0
    if(mbcsTable->outputType==MBCS_OUTPUT_1) {
878
0
        const uint16_t *stage2, *stage3, *results;
879
0
        uint16_t minValue;
880
881
0
        results=(const uint16_t *)mbcsTable->fromUnicodeBytes;
882
883
        /*
884
         * Set a threshold variable for selecting which mappings to use.
885
         * See ucnv_MBCSSingleFromBMPWithOffsets() and
886
         * MBCS_SINGLE_RESULT_FROM_U() for details.
887
         */
888
0
        if(which==UCNV_ROUNDTRIP_SET) {
889
            /* use only roundtrips */
890
0
            minValue=0xf00;
891
0
        } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ {
892
            /* use all roundtrip and fallback results */
893
0
            minValue=0x800;
894
0
        }
895
896
0
        for(st1=0; st1<maxStage1; ++st1) {
897
0
            st2=table[st1];
898
0
            if(st2>maxStage1) {
899
0
                stage2=table+st2;
900
0
                for(st2=0; st2<64; ++st2) {
901
0
                    if((st3=stage2[st2])!=0) {
902
                        /* read the stage 3 block */
903
0
                        stage3=results+st3;
904
905
0
                        do {
906
0
                            if(*stage3++>=minValue) {
907
0
                                sa->add(sa->set, c);
908
0
                            }
909
0
                        } while((++c&0xf)!=0);
910
0
                    } else {
911
0
                        c+=16; /* empty stage 3 block */
912
0
                    }
913
0
                }
914
0
            } else {
915
0
                c+=1024; /* empty stage 2 block */
916
0
            }
917
0
        }
918
0
    } else {
919
0
        const uint32_t *stage2;
920
0
        const uint8_t *stage3, *bytes;
921
0
        uint32_t st3Multiplier;
922
0
        uint32_t value;
923
0
        UBool useFallback;
924
925
0
        bytes=mbcsTable->fromUnicodeBytes;
926
927
0
        useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET);
928
929
0
        switch(mbcsTable->outputType) {
930
0
        case MBCS_OUTPUT_3:
931
0
        case MBCS_OUTPUT_4_EUC:
932
0
            st3Multiplier=3;
933
0
            break;
934
0
        case MBCS_OUTPUT_4:
935
0
            st3Multiplier=4;
936
0
            break;
937
0
        default:
938
0
            st3Multiplier=2;
939
0
            break;
940
0
        }
941
942
0
        for(st1=0; st1<maxStage1; ++st1) {
943
0
            st2=table[st1];
944
0
            if(st2>(maxStage1>>1)) {
945
0
                stage2=(const uint32_t *)table+st2;
946
0
                for(st2=0; st2<64; ++st2) {
947
0
                    if((st3=stage2[st2])!=0) {
948
                        /* read the stage 3 block */
949
0
                        stage3=bytes+st3Multiplier*16*(uint32_t)(uint16_t)st3;
950
951
                        /* get the roundtrip flags for the stage 3 block */
952
0
                        st3>>=16;
953
954
                        /*
955
                         * Add code points for which the roundtrip flag is set,
956
                         * or which map to non-zero bytes if we use fallbacks.
957
                         * See ucnv_MBCSFromUnicodeWithOffsets() for details.
958
                         */
959
0
                        switch(filter) {
960
0
                        case UCNV_SET_FILTER_NONE:
961
0
                            do {
962
0
                                if(st3&1) {
963
0
                                    sa->add(sa->set, c);
964
0
                                    stage3+=st3Multiplier;
965
0
                                } else if(useFallback) {
966
0
                                    uint8_t b=0;
967
0
                                    switch(st3Multiplier) {
968
0
                                    case 4:
969
0
                                        b|=*stage3++;
970
0
                                        U_FALLTHROUGH;
971
0
                                    case 3:
972
0
                                        b|=*stage3++;
973
0
                                        U_FALLTHROUGH;
974
0
                                    case 2:
975
0
                                        b|=stage3[0]|stage3[1];
976
0
                                        stage3+=2;
977
0
                                        U_FALLTHROUGH;
978
0
                                    default:
979
0
                                        break;
980
0
                                    }
981
0
                                    if(b!=0) {
982
0
                                        sa->add(sa->set, c);
983
0
                                    }
984
0
                                }
985
0
                                st3>>=1;
986
0
                            } while((++c&0xf)!=0);
987
0
                            break;
988
0
                        case UCNV_SET_FILTER_DBCS_ONLY:
989
                             /* Ignore single-byte results (<0x100). */
990
0
                            do {
991
0
                                if(((st3&1)!=0 || useFallback) && *((const uint16_t *)stage3)>=0x100) {
992
0
                                    sa->add(sa->set, c);
993
0
                                }
994
0
                                st3>>=1;
995
0
                                stage3+=2;  /* +=st3Multiplier */
996
0
                            } while((++c&0xf)!=0);
997
0
                            break;
998
0
                        case UCNV_SET_FILTER_2022_CN:
999
                             /* Only add code points that map to CNS 11643 planes 1 & 2 for non-EXT ISO-2022-CN. */
1000
0
                            do {
1001
0
                                if(((st3&1)!=0 || useFallback) && ((value=*stage3)==0x81 || value==0x82)) {
1002
0
                                    sa->add(sa->set, c);
1003
0
                                }
1004
0
                                st3>>=1;
1005
0
                                stage3+=3;  /* +=st3Multiplier */
1006
0
                            } while((++c&0xf)!=0);
1007
0
                            break;
1008
0
                        case UCNV_SET_FILTER_SJIS:
1009
                             /* Only add code points that map to Shift-JIS codes corresponding to JIS X 0208. */
1010
0
                            do {
1011
0
                                if(((st3&1)!=0 || useFallback) && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) {
1012
0
                                    sa->add(sa->set, c);
1013
0
                                }
1014
0
                                st3>>=1;
1015
0
                                stage3+=2;  /* +=st3Multiplier */
1016
0
                            } while((++c&0xf)!=0);
1017
0
                            break;
1018
0
                        case UCNV_SET_FILTER_GR94DBCS:
1019
                            /* Only add code points that map to ISO 2022 GR 94 DBCS codes (each byte A1..FE). */
1020
0
                            do {
1021
0
                                if( ((st3&1)!=0 || useFallback) &&
1022
0
                                    (uint16_t)((value=*((const uint16_t *)stage3)) - 0xa1a1)<=(0xfefe - 0xa1a1) &&
1023
0
                                    (uint8_t)(value-0xa1)<=(0xfe - 0xa1)
1024
0
                                ) {
1025
0
                                    sa->add(sa->set, c);
1026
0
                                }
1027
0
                                st3>>=1;
1028
0
                                stage3+=2;  /* +=st3Multiplier */
1029
0
                            } while((++c&0xf)!=0);
1030
0
                            break;
1031
0
                        case UCNV_SET_FILTER_HZ:
1032
                            /* Only add code points that are suitable for HZ DBCS (lead byte A1..FD). */
1033
0
                            do {
1034
0
                                if( ((st3&1)!=0 || useFallback) &&
1035
0
                                    (uint16_t)((value=*((const uint16_t *)stage3))-0xa1a1)<=(0xfdfe - 0xa1a1) &&
1036
0
                                    (uint8_t)(value-0xa1)<=(0xfe - 0xa1)
1037
0
                                ) {
1038
0
                                    sa->add(sa->set, c);
1039
0
                                }
1040
0
                                st3>>=1;
1041
0
                                stage3+=2;  /* +=st3Multiplier */
1042
0
                            } while((++c&0xf)!=0);
1043
0
                            break;
1044
0
                        default:
1045
0
                            *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1046
0
                            return;
1047
0
                        }
1048
0
                    } else {
1049
0
                        c+=16; /* empty stage 3 block */
1050
0
                    }
1051
0
                }
1052
0
            } else {
1053
0
                c+=1024; /* empty stage 2 block */
1054
0
            }
1055
0
        }
1056
0
    }
1057
1058
0
    ucnv_extGetUnicodeSet(sharedData, sa, which, filter, pErrorCode);
1059
0
}
1060
1061
U_CFUNC void
1062
ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
1063
                                 const USetAdder *sa,
1064
                                 UConverterUnicodeSet which,
1065
0
                                 UErrorCode *pErrorCode) {
1066
0
    ucnv_MBCSGetFilteredUnicodeSetForUnicode(
1067
0
        sharedData, sa, which,
1068
0
        sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ?
1069
0
            UCNV_SET_FILTER_DBCS_ONLY :
1070
0
            UCNV_SET_FILTER_NONE,
1071
0
        pErrorCode);
1072
0
}
1073
1074
static void U_CALLCONV
1075
ucnv_MBCSGetUnicodeSet(const UConverter *cnv,
1076
                   const USetAdder *sa,
1077
                   UConverterUnicodeSet which,
1078
0
                   UErrorCode *pErrorCode) {
1079
0
    if(cnv->options&_MBCS_OPTION_GB18030) {
1080
0
        sa->addRange(sa->set, 0, 0xd7ff);
1081
0
        sa->addRange(sa->set, 0xe000, 0x10ffff);
1082
0
    } else {
1083
0
        ucnv_MBCSGetUnicodeSetForUnicode(cnv->sharedData, sa, which, pErrorCode);
1084
0
    }
1085
0
}
1086
1087
/* conversion extensions for input not in the main table -------------------- */
1088
1089
/*
1090
 * Hardcoded extension handling for GB 18030.
1091
 * Definition of LINEAR macros and gb18030Ranges see near the beginning of the file.
1092
 *
1093
 * In the future, conversion extensions may handle m:n mappings and delta tables,
1094
 * see https://htmlpreview.github.io/?https://github.com/unicode-org/icu-docs/blob/main/design/conversion/conversion_extensions.html
1095
 *
1096
 * If an input character cannot be mapped, then these functions set an error
1097
 * code. The framework will then call the callback function.
1098
 */
1099
1100
/*
1101
 * @return if(U_FAILURE) return the code point for cnv->fromUChar32
1102
 *         else return 0 after output has been written to the target
1103
 */
1104
static UChar32
1105
_extFromU(UConverter *cnv, const UConverterSharedData *sharedData,
1106
          UChar32 cp,
1107
          const UChar **source, const UChar *sourceLimit,
1108
          uint8_t **target, const uint8_t *targetLimit,
1109
          int32_t **offsets, int32_t sourceIndex,
1110
          UBool flush,
1111
0
          UErrorCode *pErrorCode) {
1112
0
    const int32_t *cx;
1113
1114
0
    cnv->useSubChar1=FALSE;
1115
1116
0
    if( (cx=sharedData->mbcs.extIndexes)!=NULL &&
1117
0
        ucnv_extInitialMatchFromU(
1118
0
            cnv, cx,
1119
0
            cp, source, sourceLimit,
1120
0
            (char **)target, (char *)targetLimit,
1121
0
            offsets, sourceIndex,
1122
0
            flush,
1123
0
            pErrorCode)
1124
0
    ) {
1125
0
        return 0; /* an extension mapping handled the input */
1126
0
    }
1127
1128
    /* GB 18030 */
1129
0
    if((cnv->options&_MBCS_OPTION_GB18030)!=0) {
1130
0
        const uint32_t *range;
1131
0
        int32_t i;
1132
1133
0
        range=gb18030Ranges[0];
1134
0
        for(i=0; i<UPRV_LENGTHOF(gb18030Ranges); range+=4, ++i) {
1135
0
            if(range[0]<=(uint32_t)cp && (uint32_t)cp<=range[1]) {
1136
                /* found the Unicode code point, output the four-byte sequence for it */
1137
0
                uint32_t linear;
1138
0
                char bytes[4];
1139
1140
                /* get the linear value of the first GB 18030 code in this range */
1141
0
                linear=range[2]-LINEAR_18030_BASE;
1142
1143
                /* add the offset from the beginning of the range */
1144
0
                linear+=((uint32_t)cp-range[0]);
1145
1146
                /* turn this into a four-byte sequence */
1147
0
                bytes[3]=(char)(0x30+linear%10); linear/=10;
1148
0
                bytes[2]=(char)(0x81+linear%126); linear/=126;
1149
0
                bytes[1]=(char)(0x30+linear%10); linear/=10;
1150
0
                bytes[0]=(char)(0x81+linear);
1151
1152
                /* output this sequence */
1153
0
                ucnv_fromUWriteBytes(cnv,
1154
0
                                     bytes, 4, (char **)target, (char *)targetLimit,
1155
0
                                     offsets, sourceIndex, pErrorCode);
1156
0
                return 0;
1157
0
            }
1158
0
        }
1159
0
    }
1160
1161
    /* no mapping */
1162
0
    *pErrorCode=U_INVALID_CHAR_FOUND;
1163
0
    return cp;
1164
0
}
1165
1166
/*
1167
 * Input sequence: cnv->toUBytes[0..length[
1168
 * @return if(U_FAILURE) return the length (toULength, byteIndex) for the input
1169
 *         else return 0 after output has been written to the target
1170
 */
1171
static int8_t
1172
_extToU(UConverter *cnv, const UConverterSharedData *sharedData,
1173
        int8_t length,
1174
        const uint8_t **source, const uint8_t *sourceLimit,
1175
        UChar **target, const UChar *targetLimit,
1176
        int32_t **offsets, int32_t sourceIndex,
1177
        UBool flush,
1178
0
        UErrorCode *pErrorCode) {
1179
0
    const int32_t *cx;
1180
1181
0
    if( (cx=sharedData->mbcs.extIndexes)!=NULL &&
1182
0
        ucnv_extInitialMatchToU(
1183
0
            cnv, cx,
1184
0
            length, (const char **)source, (const char *)sourceLimit,
1185
0
            target, targetLimit,
1186
0
            offsets, sourceIndex,
1187
0
            flush,
1188
0
            pErrorCode)
1189
0
    ) {
1190
0
        return 0; /* an extension mapping handled the input */
1191
0
    }
1192
1193
    /* GB 18030 */
1194
0
    if(length==4 && (cnv->options&_MBCS_OPTION_GB18030)!=0) {
1195
0
        const uint32_t *range;
1196
0
        uint32_t linear;
1197
0
        int32_t i;
1198
1199
0
        linear=LINEAR_18030(cnv->toUBytes[0], cnv->toUBytes[1], cnv->toUBytes[2], cnv->toUBytes[3]);
1200
0
        range=gb18030Ranges[0];
1201
0
        for(i=0; i<UPRV_LENGTHOF(gb18030Ranges); range+=4, ++i) {
1202
0
            if(range[2]<=linear && linear<=range[3]) {
1203
                /* found the sequence, output the Unicode code point for it */
1204
0
                *pErrorCode=U_ZERO_ERROR;
1205
1206
                /* add the linear difference between the input and start sequences to the start code point */
1207
0
                linear=range[0]+(linear-range[2]);
1208
1209
                /* output this code point */
1210
0
                ucnv_toUWriteCodePoint(cnv, linear, target, targetLimit, offsets, sourceIndex, pErrorCode);
1211
1212
0
                return 0;
1213
0
            }
1214
0
        }
1215
0
    }
1216
1217
    /* no mapping */
1218
0
    *pErrorCode=U_INVALID_CHAR_FOUND;
1219
0
    return length;
1220
0
}
1221
1222
/* EBCDIC swap LF<->NL ------------------------------------------------------ */
1223
1224
/*
1225
 * This code modifies a standard EBCDIC<->Unicode mapping table for
1226
 * OS/390 (z/OS) Unix System Services (Open Edition).
1227
 * The difference is in the mapping of Line Feed and New Line control codes:
1228
 * Standard EBCDIC maps
1229
 *
1230
 *   <U000A> \x25 |0
1231
 *   <U0085> \x15 |0
1232
 *
1233
 * but OS/390 USS EBCDIC swaps the control codes for LF and NL,
1234
 * mapping
1235
 *
1236
 *   <U000A> \x15 |0
1237
 *   <U0085> \x25 |0
1238
 *
1239
 * This code modifies a loaded standard EBCDIC<->Unicode mapping table
1240
 * by copying it into allocated memory and swapping the LF and NL values.
1241
 * It allows to support the same EBCDIC charset in both versions without
1242
 * duplicating the entire installed table.
1243
 */
1244
1245
/* standard EBCDIC codes */
1246
0
#define EBCDIC_LF 0x25
1247
0
#define EBCDIC_NL 0x15
1248
1249
/* standard EBCDIC codes with roundtrip flag as stored in Unicode-to-single-byte tables */
1250
0
#define EBCDIC_RT_LF 0xf25
1251
0
#define EBCDIC_RT_NL 0xf15
1252
1253
/* Unicode code points */
1254
#define U_LF 0x0a
1255
#define U_NL 0x85
1256
1257
static UBool
1258
0
_EBCDICSwapLFNL(UConverterSharedData *sharedData, UErrorCode *pErrorCode) {
1259
0
    UConverterMBCSTable *mbcsTable;
1260
1261
0
    const uint16_t *table, *results;
1262
0
    const uint8_t *bytes;
1263
1264
0
    int32_t (*newStateTable)[256];
1265
0
    uint16_t *newResults;
1266
0
    uint8_t *p;
1267
0
    char *name;
1268
1269
0
    uint32_t stage2Entry;
1270
0
    uint32_t size, sizeofFromUBytes;
1271
1272
0
    mbcsTable=&sharedData->mbcs;
1273
1274
0
    table=mbcsTable->fromUnicodeTable;
1275
0
    bytes=mbcsTable->fromUnicodeBytes;
1276
0
    results=(const uint16_t *)bytes;
1277
1278
    /*
1279
     * Check that this is an EBCDIC table with SBCS portion -
1280
     * SBCS or EBCDIC_STATEFUL with standard EBCDIC LF and NL mappings.
1281
     *
1282
     * If not, ignore the option. Options are always ignored if they do not apply.
1283
     */
1284
0
    if(!(
1285
0
         (mbcsTable->outputType==MBCS_OUTPUT_1 || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) &&
1286
0
         mbcsTable->stateTable[0][EBCDIC_LF]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF) &&
1287
0
         mbcsTable->stateTable[0][EBCDIC_NL]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL)
1288
0
    )) {
1289
0
        return FALSE;
1290
0
    }
1291
1292
0
    if(mbcsTable->outputType==MBCS_OUTPUT_1) {
1293
0
        if(!(
1294
0
             EBCDIC_RT_LF==MBCS_SINGLE_RESULT_FROM_U(table, results, U_LF) &&
1295
0
             EBCDIC_RT_NL==MBCS_SINGLE_RESULT_FROM_U(table, results, U_NL)
1296
0
        )) {
1297
0
            return FALSE;
1298
0
        }
1299
0
    } else /* MBCS_OUTPUT_2_SISO */ {
1300
0
        stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF);
1301
0
        if(!(
1302
0
             MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_LF)!=0 &&
1303
0
             EBCDIC_LF==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_LF)
1304
0
        )) {
1305
0
            return FALSE;
1306
0
        }
1307
1308
0
        stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL);
1309
0
        if(!(
1310
0
             MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_NL)!=0 &&
1311
0
             EBCDIC_NL==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_NL)
1312
0
        )) {
1313
0
            return FALSE;
1314
0
        }
1315
0
    }
1316
1317
0
    if(mbcsTable->fromUBytesLength>0) {
1318
        /*
1319
         * We _know_ the number of bytes in the fromUnicodeBytes array
1320
         * starting with header.version 4.1.
1321
         */
1322
0
        sizeofFromUBytes=mbcsTable->fromUBytesLength;
1323
0
    } else {
1324
        /*
1325
         * Otherwise:
1326
         * There used to be code to enumerate the fromUnicode
1327
         * trie and find the highest entry, but it was removed in ICU 3.2
1328
         * because it was not tested and caused a low code coverage number.
1329
         * See Jitterbug 3674.
1330
         * This affects only some .cnv file formats with a header.version
1331
         * below 4.1, and only when swaplfnl is requested.
1332
         *
1333
         * ucnvmbcs.c revision 1.99 is the last one with the
1334
         * ucnv_MBCSSizeofFromUBytes() function.
1335
         */
1336
0
        *pErrorCode=U_INVALID_FORMAT_ERROR;
1337
0
        return FALSE;
1338
0
    }
1339
1340
    /*
1341
     * The table has an appropriate format.
1342
     * Allocate and build
1343
     * - a modified to-Unicode state table
1344
     * - a modified from-Unicode output array
1345
     * - a converter name string with the swap option appended
1346
     */
1347
0
    size=
1348
0
        mbcsTable->countStates*1024+
1349
0
        sizeofFromUBytes+
1350
0
        UCNV_MAX_CONVERTER_NAME_LENGTH+20;
1351
0
    p=(uint8_t *)uprv_malloc(size);
1352
0
    if(p==NULL) {
1353
0
        *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
1354
0
        return FALSE;
1355
0
    }
1356
1357
    /* copy and modify the to-Unicode state table */
1358
0
    newStateTable=(int32_t (*)[256])p;
1359
0
    uprv_memcpy(newStateTable, mbcsTable->stateTable, mbcsTable->countStates*1024);
1360
1361
0
    newStateTable[0][EBCDIC_LF]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL);
1362
0
    newStateTable[0][EBCDIC_NL]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF);
1363
1364
    /* copy and modify the from-Unicode result table */
1365
0
    newResults=(uint16_t *)newStateTable[mbcsTable->countStates];
1366
0
    uprv_memcpy(newResults, bytes, sizeofFromUBytes);
1367
1368
    /* conveniently, the table access macros work on the left side of expressions */
1369
0
    if(mbcsTable->outputType==MBCS_OUTPUT_1) {
1370
0
        MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_LF)=EBCDIC_RT_NL;
1371
0
        MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_NL)=EBCDIC_RT_LF;
1372
0
    } else /* MBCS_OUTPUT_2_SISO */ {
1373
0
        stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF);
1374
0
        MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_LF)=EBCDIC_NL;
1375
1376
0
        stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL);
1377
0
        MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_NL)=EBCDIC_LF;
1378
0
    }
1379
1380
    /* set the canonical converter name */
1381
0
    name=(char *)newResults+sizeofFromUBytes;
1382
0
    uprv_strcpy(name, sharedData->staticData->name);
1383
0
    uprv_strcat(name, UCNV_SWAP_LFNL_OPTION_STRING);
1384
1385
    /* set the pointers */
1386
0
    icu::umtx_lock(NULL);
1387
0
    if(mbcsTable->swapLFNLStateTable==NULL) {
1388
0
        mbcsTable->swapLFNLStateTable=newStateTable;
1389
0
        mbcsTable->swapLFNLFromUnicodeBytes=(uint8_t *)newResults;
1390
0
        mbcsTable->swapLFNLName=name;
1391
1392
0
        newStateTable=NULL;
1393
0
    }
1394
0
    icu::umtx_unlock(NULL);
1395
1396
    /* release the allocated memory if another thread beat us to it */
1397
0
    if(newStateTable!=NULL) {
1398
0
        uprv_free(newStateTable);
1399
0
    }
1400
0
    return TRUE;
1401
0
}
1402
1403
/* reconstitute omitted fromUnicode data ------------------------------------ */
1404
1405
/* for details, compare with genmbcs.c MBCSAddFromUnicode() and transformEUC() */
1406
static UBool U_CALLCONV
1407
0
writeStage3Roundtrip(const void *context, uint32_t value, UChar32 codePoints[32]) {
1408
0
    UConverterMBCSTable *mbcsTable=(UConverterMBCSTable *)context;
1409
0
    const uint16_t *table;
1410
0
    uint32_t *stage2;
1411
0
    uint8_t *bytes, *p;
1412
0
    UChar32 c;
1413
0
    int32_t i, st3;
1414
1415
0
    table=mbcsTable->fromUnicodeTable;
1416
0
    bytes=(uint8_t *)mbcsTable->fromUnicodeBytes;
1417
1418
    /* for EUC outputTypes, modify the value like genmbcs.c's transformEUC() */
1419
0
    switch(mbcsTable->outputType) {
1420
0
    case MBCS_OUTPUT_3_EUC:
1421
0
        if(value<=0xffff) {
1422
            /* short sequences are stored directly */
1423
            /* code set 0 or 1 */
1424
0
        } else if(value<=0x8effff) {
1425
            /* code set 2 */
1426
0
            value&=0x7fff;
1427
0
        } else /* first byte is 0x8f */ {
1428
            /* code set 3 */
1429
0
            value&=0xff7f;
1430
0
        }
1431
0
        break;
1432
0
    case MBCS_OUTPUT_4_EUC:
1433
0
        if(value<=0xffffff) {
1434
            /* short sequences are stored directly */
1435
            /* code set 0 or 1 */
1436
0
        } else if(value<=0x8effffff) {
1437
            /* code set 2 */
1438
0
            value&=0x7fffff;
1439
0
        } else /* first byte is 0x8f */ {
1440
            /* code set 3 */
1441
0
            value&=0xff7fff;
1442
0
        }
1443
0
        break;
1444
0
    default:
1445
0
        break;
1446
0
    }
1447
1448
0
    for(i=0; i<=0x1f; ++value, ++i) {
1449
0
        c=codePoints[i];
1450
0
        if(c<0) {
1451
0
            continue;
1452
0
        }
1453
1454
        /* locate the stage 2 & 3 data */
1455
0
        stage2=((uint32_t *)table)+table[c>>10]+((c>>4)&0x3f);
1456
0
        p=bytes;
1457
0
        st3=(int32_t)(uint16_t)*stage2*16+(c&0xf);
1458
1459
        /* write the codepage bytes into stage 3 */
1460
0
        switch(mbcsTable->outputType) {
1461
0
        case MBCS_OUTPUT_3:
1462
0
        case MBCS_OUTPUT_4_EUC:
1463
0
            p+=st3*3;
1464
0
            p[0]=(uint8_t)(value>>16);
1465
0
            p[1]=(uint8_t)(value>>8);
1466
0
            p[2]=(uint8_t)value;
1467
0
            break;
1468
0
        case MBCS_OUTPUT_4:
1469
0
            ((uint32_t *)p)[st3]=value;
1470
0
            break;
1471
0
        default:
1472
            /* 2 bytes per character */
1473
0
            ((uint16_t *)p)[st3]=(uint16_t)value;
1474
0
            break;
1475
0
        }
1476
1477
        /* set the roundtrip flag */
1478
0
        *stage2|=(1UL<<(16+(c&0xf)));
1479
0
    }
1480
0
    return TRUE;
1481
0
 }
1482
1483
static void
1484
reconstituteData(UConverterMBCSTable *mbcsTable,
1485
                 uint32_t stage1Length, uint32_t stage2Length,
1486
                 uint32_t fullStage2Length,  /* lengths are numbers of units, not bytes */
1487
0
                 UErrorCode *pErrorCode) {
1488
0
    uint16_t *stage1;
1489
0
    uint32_t *stage2;
1490
0
    uint32_t dataLength=stage1Length*2+fullStage2Length*4+mbcsTable->fromUBytesLength;
1491
0
    mbcsTable->reconstitutedData=(uint8_t *)uprv_malloc(dataLength);
1492
0
    if(mbcsTable->reconstitutedData==NULL) {
1493
0
        *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
1494
0
        return;
1495
0
    }
1496
0
    uprv_memset(mbcsTable->reconstitutedData, 0, dataLength);
1497
1498
    /* copy existing data and reroute the pointers */
1499
0
    stage1=(uint16_t *)mbcsTable->reconstitutedData;
1500
0
    uprv_memcpy(stage1, mbcsTable->fromUnicodeTable, stage1Length*2);
1501
1502
0
    stage2=(uint32_t *)(stage1+stage1Length);
1503
0
    uprv_memcpy(stage2+(fullStage2Length-stage2Length),
1504
0
                mbcsTable->fromUnicodeTable+stage1Length,
1505
0
                stage2Length*4);
1506
1507
0
    mbcsTable->fromUnicodeTable=stage1;
1508
0
    mbcsTable->fromUnicodeBytes=(uint8_t *)(stage2+fullStage2Length);
1509
1510
    /* indexes into stage 2 count from the bottom of the fromUnicodeTable */
1511
0
    stage2=(uint32_t *)stage1;
1512
1513
    /* reconstitute the initial part of stage 2 from the mbcsIndex */
1514
0
    {
1515
0
        int32_t stageUTF8Length=((int32_t)mbcsTable->maxFastUChar+1)>>6;
1516
0
        int32_t stageUTF8Index=0;
1517
0
        int32_t st1, st2, st3, i;
1518
1519
0
        for(st1=0; stageUTF8Index<stageUTF8Length; ++st1) {
1520
0
            st2=stage1[st1];
1521
0
            if(st2!=(int32_t)stage1Length/2) {
1522
                /* each stage 2 block has 64 entries corresponding to 16 entries in the mbcsIndex */
1523
0
                for(i=0; i<16; ++i) {
1524
0
                    st3=mbcsTable->mbcsIndex[stageUTF8Index++];
1525
0
                    if(st3!=0) {
1526
                        /* an stage 2 entry's index is per stage 3 16-block, not per stage 3 entry */
1527
0
                        st3>>=4;
1528
                        /*
1529
                         * 4 stage 2 entries point to 4 consecutive stage 3 16-blocks which are
1530
                         * allocated together as a single 64-block for access from the mbcsIndex
1531
                         */
1532
0
                        stage2[st2++]=st3++;
1533
0
                        stage2[st2++]=st3++;
1534
0
                        stage2[st2++]=st3++;
1535
0
                        stage2[st2++]=st3;
1536
0
                    } else {
1537
                        /* no stage 3 block, skip */
1538
0
                        st2+=4;
1539
0
                    }
1540
0
                }
1541
0
            } else {
1542
                /* no stage 2 block, skip */
1543
0
                stageUTF8Index+=16;
1544
0
            }
1545
0
        }
1546
0
    }
1547
1548
    /* reconstitute fromUnicodeBytes with roundtrips from toUnicode data */
1549
0
    ucnv_MBCSEnumToUnicode(mbcsTable, writeStage3Roundtrip, mbcsTable, pErrorCode);
1550
0
}
1551
1552
/* MBCS setup functions ----------------------------------------------------- */
1553
1554
static void U_CALLCONV
1555
ucnv_MBCSLoad(UConverterSharedData *sharedData,
1556
          UConverterLoadArgs *pArgs,
1557
          const uint8_t *raw,
1558
0
          UErrorCode *pErrorCode) {
1559
0
    UDataInfo info;
1560
0
    UConverterMBCSTable *mbcsTable=&sharedData->mbcs;
1561
0
    _MBCSHeader *header=(_MBCSHeader *)raw;
1562
0
    uint32_t offset;
1563
0
    uint32_t headerLength;
1564
0
    UBool noFromU=FALSE;
1565
1566
0
    if(header->version[0]==4) {
1567
0
        headerLength=MBCS_HEADER_V4_LENGTH;
1568
0
    } else if(header->version[0]==5 && header->version[1]>=3 &&
1569
0
              (header->options&MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK)==0) {
1570
0
        headerLength=header->options&MBCS_OPT_LENGTH_MASK;
1571
0
        noFromU=(UBool)((header->options&MBCS_OPT_NO_FROM_U)!=0);
1572
0
    } else {
1573
0
        *pErrorCode=U_INVALID_TABLE_FORMAT;
1574
0
        return;
1575
0
    }
1576
1577
0
    mbcsTable->outputType=(uint8_t)header->flags;
1578
0
    if(noFromU && mbcsTable->outputType==MBCS_OUTPUT_1) {
1579
0
        *pErrorCode=U_INVALID_TABLE_FORMAT;
1580
0
        return;
1581
0
    }
1582
1583
    /* extension data, header version 4.2 and higher */
1584
0
    offset=header->flags>>8;
1585
0
    if(offset!=0) {
1586
0
        mbcsTable->extIndexes=(const int32_t *)(raw+offset);
1587
0
    }
1588
1589
0
    if(mbcsTable->outputType==MBCS_OUTPUT_EXT_ONLY) {
1590
0
        UConverterLoadArgs args=UCNV_LOAD_ARGS_INITIALIZER;
1591
0
        UConverterSharedData *baseSharedData;
1592
0
        const int32_t *extIndexes;
1593
0
        const char *baseName;
1594
1595
        /* extension-only file, load the base table and set values appropriately */
1596
0
        if((extIndexes=mbcsTable->extIndexes)==NULL) {
1597
            /* extension-only file without extension */
1598
0
            *pErrorCode=U_INVALID_TABLE_FORMAT;
1599
0
            return;
1600
0
        }
1601
1602
0
        if(pArgs->nestedLoads!=1) {
1603
            /* an extension table must not be loaded as a base table */
1604
0
            *pErrorCode=U_INVALID_TABLE_FILE;
1605
0
            return;
1606
0
        }
1607
1608
        /* load the base table */
1609
0
        baseName=(const char *)header+headerLength*4;
1610
0
        if(0==uprv_strcmp(baseName, sharedData->staticData->name)) {
1611
            /* forbid loading this same extension-only file */
1612
0
            *pErrorCode=U_INVALID_TABLE_FORMAT;
1613
0
            return;
1614
0
        }
1615
1616
        /* TODO parse package name out of the prefix of the base name in the extension .cnv file? */
1617
0
        args.size=sizeof(UConverterLoadArgs);
1618
0
        args.nestedLoads=2;
1619
0
        args.onlyTestIsLoadable=pArgs->onlyTestIsLoadable;
1620
0
        args.reserved=pArgs->reserved;
1621
0
        args.options=pArgs->options;
1622
0
        args.pkg=pArgs->pkg;
1623
0
        args.name=baseName;
1624
0
        baseSharedData=ucnv_load(&args, pErrorCode);
1625
0
        if(U_FAILURE(*pErrorCode)) {
1626
0
            return;
1627
0
        }
1628
0
        if( baseSharedData->staticData->conversionType!=UCNV_MBCS ||
1629
0
            baseSharedData->mbcs.baseSharedData!=NULL
1630
0
        ) {
1631
0
            ucnv_unload(baseSharedData);
1632
0
            *pErrorCode=U_INVALID_TABLE_FORMAT;
1633
0
            return;
1634
0
        }
1635
0
        if(pArgs->onlyTestIsLoadable) {
1636
            /*
1637
             * Exit as soon as we know that we can load the converter
1638
             * and the format is valid and supported.
1639
             * The worst that can happen in the following code is a memory
1640
             * allocation error.
1641
             */
1642
0
            ucnv_unload(baseSharedData);
1643
0
            return;
1644
0
        }
1645
1646
        /* copy the base table data */
1647
0
        uprv_memcpy(mbcsTable, &baseSharedData->mbcs, sizeof(UConverterMBCSTable));
1648
1649
        /* overwrite values with relevant ones for the extension converter */
1650
0
        mbcsTable->baseSharedData=baseSharedData;
1651
0
        mbcsTable->extIndexes=extIndexes;
1652
1653
        /*
1654
         * It would be possible to share the swapLFNL data with a base converter,
1655
         * but the generated name would have to be different, and the memory
1656
         * would have to be free'd only once.
1657
         * It is easier to just create the data for the extension converter
1658
         * separately when it is requested.
1659
         */
1660
0
        mbcsTable->swapLFNLStateTable=NULL;
1661
0
        mbcsTable->swapLFNLFromUnicodeBytes=NULL;
1662
0
        mbcsTable->swapLFNLName=NULL;
1663
1664
        /*
1665
         * The reconstitutedData must be deleted only when the base converter
1666
         * is unloaded.
1667
         */
1668
0
        mbcsTable->reconstitutedData=NULL;
1669
1670
        /*
1671
         * Set a special, runtime-only outputType if the extension converter
1672
         * is a DBCS version of a base converter that also maps single bytes.
1673
         */
1674
0
        if( sharedData->staticData->conversionType==UCNV_DBCS ||
1675
0
                (sharedData->staticData->conversionType==UCNV_MBCS &&
1676
0
                 sharedData->staticData->minBytesPerChar>=2)
1677
0
        ) {
1678
0
            if(baseSharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO) {
1679
                /* the base converter is SI/SO-stateful */
1680
0
                int32_t entry;
1681
1682
                /* get the dbcs state from the state table entry for SO=0x0e */
1683
0
                entry=mbcsTable->stateTable[0][0xe];
1684
0
                if( MBCS_ENTRY_IS_FINAL(entry) &&
1685
0
                    MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_CHANGE_ONLY &&
1686
0
                    MBCS_ENTRY_FINAL_STATE(entry)!=0
1687
0
                ) {
1688
0
                    mbcsTable->dbcsOnlyState=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry);
1689
1690
0
                    mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY;
1691
0
                }
1692
0
            } else if(
1693
0
                baseSharedData->staticData->conversionType==UCNV_MBCS &&
1694
0
                baseSharedData->staticData->minBytesPerChar==1 &&
1695
0
                baseSharedData->staticData->maxBytesPerChar==2 &&
1696
0
                mbcsTable->countStates<=127
1697
0
            ) {
1698
                /* non-stateful base converter, need to modify the state table */
1699
0
                int32_t (*newStateTable)[256];
1700
0
                int32_t *state;
1701
0
                int32_t i, count;
1702
1703
                /* allocate a new state table and copy the base state table contents */
1704
0
                count=mbcsTable->countStates;
1705
0
                newStateTable=(int32_t (*)[256])uprv_malloc((count+1)*1024);
1706
0
                if(newStateTable==NULL) {
1707
0
                    ucnv_unload(baseSharedData);
1708
0
                    *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
1709
0
                    return;
1710
0
                }
1711
1712
0
                uprv_memcpy(newStateTable, mbcsTable->stateTable, count*1024);
1713
1714
                /* change all final single-byte entries to go to a new all-illegal state */
1715
0
                state=newStateTable[0];
1716
0
                for(i=0; i<256; ++i) {
1717
0
                    if(MBCS_ENTRY_IS_FINAL(state[i])) {
1718
0
                        state[i]=MBCS_ENTRY_TRANSITION(count, 0);
1719
0
                    }
1720
0
                }
1721
1722
                /* build the new all-illegal state */
1723
0
                state=newStateTable[count];
1724
0
                for(i=0; i<256; ++i) {
1725
0
                    state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0);
1726
0
                }
1727
0
                mbcsTable->stateTable=(const int32_t (*)[256])newStateTable;
1728
0
                mbcsTable->countStates=(uint8_t)(count+1);
1729
0
                mbcsTable->stateTableOwned=TRUE;
1730
1731
0
                mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY;
1732
0
            }
1733
0
        }
1734
1735
        /*
1736
         * unlike below for files with base tables, do not get the unicodeMask
1737
         * from the sharedData; instead, use the base table's unicodeMask,
1738
         * which we copied in the memcpy above;
1739
         * this is necessary because the static data unicodeMask, especially
1740
         * the UCNV_HAS_SUPPLEMENTARY flag, is part of the base table data
1741
         */
1742
0
    } else {
1743
        /* conversion file with a base table; an additional extension table is optional */
1744
        /* make sure that the output type is known */
1745
0
        switch(mbcsTable->outputType) {
1746
0
        case MBCS_OUTPUT_1:
1747
0
        case MBCS_OUTPUT_2:
1748
0
        case MBCS_OUTPUT_3:
1749
0
        case MBCS_OUTPUT_4:
1750
0
        case MBCS_OUTPUT_3_EUC:
1751
0
        case MBCS_OUTPUT_4_EUC:
1752
0
        case MBCS_OUTPUT_2_SISO:
1753
            /* OK */
1754
0
            break;
1755
0
        default:
1756
0
            *pErrorCode=U_INVALID_TABLE_FORMAT;
1757
0
            return;
1758
0
        }
1759
0
        if(pArgs->onlyTestIsLoadable) {
1760
            /*
1761
             * Exit as soon as we know that we can load the converter
1762
             * and the format is valid and supported.
1763
             * The worst that can happen in the following code is a memory
1764
             * allocation error.
1765
             */
1766
0
            return;
1767
0
        }
1768
1769
0
        mbcsTable->countStates=(uint8_t)header->countStates;
1770
0
        mbcsTable->countToUFallbacks=header->countToUFallbacks;
1771
0
        mbcsTable->stateTable=(const int32_t (*)[256])(raw+headerLength*4);
1772
0
        mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates);
1773
0
        mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits);
1774
1775
0
        mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable);
1776
0
        mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes);
1777
0
        mbcsTable->fromUBytesLength=header->fromUBytesLength;
1778
1779
        /*
1780
         * converter versions 6.1 and up contain a unicodeMask that is
1781
         * used here to select the most efficient function implementations
1782
         */
1783
0
        info.size=sizeof(UDataInfo);
1784
0
        udata_getInfo((UDataMemory *)sharedData->dataMemory, &info);
1785
0
        if(info.formatVersion[0]>6 || (info.formatVersion[0]==6 && info.formatVersion[1]>=1)) {
1786
            /* mask off possible future extensions to be safe */
1787
0
            mbcsTable->unicodeMask=(uint8_t)(sharedData->staticData->unicodeMask&3);
1788
0
        } else {
1789
            /* for older versions, assume worst case: contains anything possible (prevent over-optimizations) */
1790
0
            mbcsTable->unicodeMask=UCNV_HAS_SUPPLEMENTARY|UCNV_HAS_SURROGATES;
1791
0
        }
1792
1793
        /*
1794
         * _MBCSHeader.version 4.3 adds utf8Friendly data structures.
1795
         * Check for the header version, SBCS vs. MBCS, and for whether the
1796
         * data structures are optimized for code points as high as what the
1797
         * runtime code is designed for.
1798
         * The implementation does not handle mapping tables with entries for
1799
         * unpaired surrogates.
1800
         */
1801
0
        if( header->version[1]>=3 &&
1802
0
            (mbcsTable->unicodeMask&UCNV_HAS_SURROGATES)==0 &&
1803
0
            (mbcsTable->countStates==1 ?
1804
0
                (header->version[2]>=(SBCS_FAST_MAX>>8)) :
1805
0
                (header->version[2]>=(MBCS_FAST_MAX>>8))
1806
0
            )
1807
0
        ) {
1808
0
            mbcsTable->utf8Friendly=TRUE;
1809
1810
0
            if(mbcsTable->countStates==1) {
1811
                /*
1812
                 * SBCS: Stage 3 is allocated in 64-entry blocks for U+0000..SBCS_FAST_MAX or higher.
1813
                 * Build a table with indexes to each block, to be used instead of
1814
                 * the regular stage 1/2 table.
1815
                 */
1816
0
                int32_t i;
1817
0
                for(i=0; i<(SBCS_FAST_LIMIT>>6); ++i) {
1818
0
                    mbcsTable->sbcsIndex[i]=mbcsTable->fromUnicodeTable[mbcsTable->fromUnicodeTable[i>>4]+((i<<2)&0x3c)];
1819
0
                }
1820
                /* set SBCS_FAST_MAX to reflect the reach of sbcsIndex[] even if header->version[2]>(SBCS_FAST_MAX>>8) */
1821
0
                mbcsTable->maxFastUChar=SBCS_FAST_MAX;
1822
0
            } else {
1823
                /*
1824
                 * MBCS: Stage 3 is allocated in 64-entry blocks for U+0000..MBCS_FAST_MAX or higher.
1825
                 * The .cnv file is prebuilt with an additional stage table with indexes
1826
                 * to each block.
1827
                 */
1828
0
                mbcsTable->mbcsIndex=(const uint16_t *)
1829
0
                    (mbcsTable->fromUnicodeBytes+
1830
0
                     (noFromU ? 0 : mbcsTable->fromUBytesLength));
1831
0
                mbcsTable->maxFastUChar=(((UChar)header->version[2])<<8)|0xff;
1832
0
            }
1833
0
        }
1834
1835
        /* calculate a bit set of 4 ASCII characters per bit that round-trip to ASCII bytes */
1836
0
        {
1837
0
            uint32_t asciiRoundtrips=0xffffffff;
1838
0
            int32_t i;
1839
1840
0
            for(i=0; i<0x80; ++i) {
1841
0
                if(mbcsTable->stateTable[0][i]!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, i)) {
1842
0
                    asciiRoundtrips&=~((uint32_t)1<<(i>>2));
1843
0
                }
1844
0
            }
1845
0
            mbcsTable->asciiRoundtrips=asciiRoundtrips;
1846
0
        }
1847
1848
0
        if(noFromU) {
1849
0
            uint32_t stage1Length=
1850
0
                mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY ?
1851
0
                    0x440 : 0x40;
1852
0
            uint32_t stage2Length=
1853
0
                (header->offsetFromUBytes-header->offsetFromUTable)/4-
1854
0
                stage1Length/2;
1855
0
            reconstituteData(mbcsTable, stage1Length, stage2Length, header->fullStage2Length, pErrorCode);
1856
0
        }
1857
0
    }
1858
1859
    /* Set the impl pointer here so that it is set for both extension-only and base tables. */
1860
0
    if(mbcsTable->utf8Friendly) {
1861
0
        if(mbcsTable->countStates==1) {
1862
0
            sharedData->impl=&_SBCSUTF8Impl;
1863
0
        } else {
1864
0
            if(mbcsTable->outputType==MBCS_OUTPUT_2) {
1865
0
                sharedData->impl=&_DBCSUTF8Impl;
1866
0
            }
1867
0
        }
1868
0
    }
1869
1870
0
    if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) {
1871
        /*
1872
         * MBCS_OUTPUT_DBCS_ONLY: No SBCS mappings, therefore ASCII does not roundtrip.
1873
         * MBCS_OUTPUT_2_SISO: Bypass the ASCII fastpath to handle prevLength correctly.
1874
         */
1875
0
        mbcsTable->asciiRoundtrips=0;
1876
0
    }
1877
0
}
1878
1879
static void U_CALLCONV
1880
0
ucnv_MBCSUnload(UConverterSharedData *sharedData) {
1881
0
    UConverterMBCSTable *mbcsTable=&sharedData->mbcs;
1882
1883
0
    if(mbcsTable->swapLFNLStateTable!=NULL) {
1884
0
        uprv_free(mbcsTable->swapLFNLStateTable);
1885
0
    }
1886
0
    if(mbcsTable->stateTableOwned) {
1887
0
        uprv_free((void *)mbcsTable->stateTable);
1888
0
    }
1889
0
    if(mbcsTable->baseSharedData!=NULL) {
1890
0
        ucnv_unload(mbcsTable->baseSharedData);
1891
0
    }
1892
0
    if(mbcsTable->reconstitutedData!=NULL) {
1893
0
        uprv_free(mbcsTable->reconstitutedData);
1894
0
    }
1895
0
}
1896
1897
static void U_CALLCONV
1898
ucnv_MBCSOpen(UConverter *cnv,
1899
              UConverterLoadArgs *pArgs,
1900
0
              UErrorCode *pErrorCode) {
1901
0
    UConverterMBCSTable *mbcsTable;
1902
0
    const int32_t *extIndexes;
1903
0
    uint8_t outputType;
1904
0
    int8_t maxBytesPerUChar;
1905
1906
0
    if(pArgs->onlyTestIsLoadable) {
1907
0
        return;
1908
0
    }
1909
1910
0
    mbcsTable=&cnv->sharedData->mbcs;
1911
0
    outputType=mbcsTable->outputType;
1912
1913
0
    if(outputType==MBCS_OUTPUT_DBCS_ONLY) {
1914
        /* the swaplfnl option does not apply, remove it */
1915
0
        cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL;
1916
0
    }
1917
1918
0
    if((pArgs->options&UCNV_OPTION_SWAP_LFNL)!=0) {
1919
        /* do this because double-checked locking is broken */
1920
0
        UBool isCached;
1921
1922
0
        icu::umtx_lock(NULL);
1923
0
        isCached=mbcsTable->swapLFNLStateTable!=NULL;
1924
0
        icu::umtx_unlock(NULL);
1925
1926
0
        if(!isCached) {
1927
0
            if(!_EBCDICSwapLFNL(cnv->sharedData, pErrorCode)) {
1928
0
                if(U_FAILURE(*pErrorCode)) {
1929
0
                    return; /* something went wrong */
1930
0
                }
1931
1932
                /* the option does not apply, remove it */
1933
0
                cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL;
1934
0
            }
1935
0
        }
1936
0
    }
1937
1938
0
    if(uprv_strstr(pArgs->name, "18030")!=NULL) {
1939
0
        if(uprv_strstr(pArgs->name, "gb18030")!=NULL || uprv_strstr(pArgs->name, "GB18030")!=NULL) {
1940
            /* set a flag for GB 18030 mode, which changes the callback behavior */
1941
0
            cnv->options|=_MBCS_OPTION_GB18030;
1942
0
        }
1943
0
    } else if((uprv_strstr(pArgs->name, "KEIS")!=NULL) || (uprv_strstr(pArgs->name, "keis")!=NULL)) {
1944
        /* set a flag for KEIS converter, which changes the SI/SO character sequence */
1945
0
        cnv->options|=_MBCS_OPTION_KEIS;
1946
0
    } else if((uprv_strstr(pArgs->name, "JEF")!=NULL) || (uprv_strstr(pArgs->name, "jef")!=NULL)) {
1947
        /* set a flag for JEF converter, which changes the SI/SO character sequence */
1948
0
        cnv->options|=_MBCS_OPTION_JEF;
1949
0
    } else if((uprv_strstr(pArgs->name, "JIPS")!=NULL) || (uprv_strstr(pArgs->name, "jips")!=NULL)) {
1950
        /* set a flag for JIPS converter, which changes the SI/SO character sequence */
1951
0
        cnv->options|=_MBCS_OPTION_JIPS;
1952
0
    }
1953
1954
    /* fix maxBytesPerUChar depending on outputType and options etc. */
1955
0
    if(outputType==MBCS_OUTPUT_2_SISO) {
1956
0
        cnv->maxBytesPerUChar=3; /* SO+DBCS */
1957
0
    }
1958
1959
0
    extIndexes=mbcsTable->extIndexes;
1960
0
    if(extIndexes!=NULL) {
1961
0
        maxBytesPerUChar=(int8_t)UCNV_GET_MAX_BYTES_PER_UCHAR(extIndexes);
1962
0
        if(outputType==MBCS_OUTPUT_2_SISO) {
1963
0
            ++maxBytesPerUChar; /* SO + multiple DBCS */
1964
0
        }
1965
1966
0
        if(maxBytesPerUChar>cnv->maxBytesPerUChar) {
1967
0
            cnv->maxBytesPerUChar=maxBytesPerUChar;
1968
0
        }
1969
0
    }
1970
1971
#if 0
1972
    /*
1973
     * documentation of UConverter fields used for status
1974
     * all of these fields are (re)set to 0 by ucnv_bld.c and ucnv_reset()
1975
     */
1976
1977
    /* toUnicode */
1978
    cnv->toUnicodeStatus=0;     /* offset */
1979
    cnv->mode=0;                /* state */
1980
    cnv->toULength=0;           /* byteIndex */
1981
1982
    /* fromUnicode */
1983
    cnv->fromUChar32=0;
1984
    cnv->fromUnicodeStatus=1;   /* prevLength */
1985
#endif
1986
0
}
1987
1988
U_CDECL_BEGIN
1989
1990
static const char* U_CALLCONV
1991
0
ucnv_MBCSGetName(const UConverter *cnv) {
1992
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0 && cnv->sharedData->mbcs.swapLFNLName!=NULL) {
1993
0
        return cnv->sharedData->mbcs.swapLFNLName;
1994
0
    } else {
1995
0
        return cnv->sharedData->staticData->name;
1996
0
    }
1997
0
}
1998
U_CDECL_END
1999
2000
2001
/* MBCS-to-Unicode conversion functions ------------------------------------- */
2002
2003
static UChar32 U_CALLCONV
2004
0
ucnv_MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) {
2005
0
    const _MBCSToUFallback *toUFallbacks;
2006
0
    uint32_t i, start, limit;
2007
2008
0
    limit=mbcsTable->countToUFallbacks;
2009
0
    if(limit>0) {
2010
        /* do a binary search for the fallback mapping */
2011
0
        toUFallbacks=mbcsTable->toUFallbacks;
2012
0
        start=0;
2013
0
        while(start<limit-1) {
2014
0
            i=(start+limit)/2;
2015
0
            if(offset<toUFallbacks[i].offset) {
2016
0
                limit=i;
2017
0
            } else {
2018
0
                start=i;
2019
0
            }
2020
0
        }
2021
2022
        /* did we really find it? */
2023
0
        if(offset==toUFallbacks[start].offset) {
2024
0
            return toUFallbacks[start].codePoint;
2025
0
        }
2026
0
    }
2027
2028
0
    return 0xfffe;
2029
0
}
2030
2031
/* This version of ucnv_MBCSToUnicodeWithOffsets() is optimized for single-byte, single-state codepages. */
2032
static void
2033
ucnv_MBCSSingleToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
2034
0
                                UErrorCode *pErrorCode) {
2035
0
    UConverter *cnv;
2036
0
    const uint8_t *source, *sourceLimit;
2037
0
    UChar *target;
2038
0
    const UChar *targetLimit;
2039
0
    int32_t *offsets;
2040
2041
0
    const int32_t (*stateTable)[256];
2042
2043
0
    int32_t sourceIndex;
2044
2045
0
    int32_t entry;
2046
0
    UChar c;
2047
0
    uint8_t action;
2048
2049
    /* set up the local pointers */
2050
0
    cnv=pArgs->converter;
2051
0
    source=(const uint8_t *)pArgs->source;
2052
0
    sourceLimit=(const uint8_t *)pArgs->sourceLimit;
2053
0
    target=pArgs->target;
2054
0
    targetLimit=pArgs->targetLimit;
2055
0
    offsets=pArgs->offsets;
2056
2057
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
2058
0
        stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
2059
0
    } else {
2060
0
        stateTable=cnv->sharedData->mbcs.stateTable;
2061
0
    }
2062
2063
    /* sourceIndex=-1 if the current character began in the previous buffer */
2064
0
    sourceIndex=0;
2065
2066
    /* conversion loop */
2067
0
    while(source<sourceLimit) {
2068
        /*
2069
         * This following test is to see if available input would overflow the output.
2070
         * It does not catch output of more than one code unit that
2071
         * overflows as a result of a surrogate pair or callback output
2072
         * from the last source byte.
2073
         * Therefore, those situations also test for overflows and will
2074
         * then break the loop, too.
2075
         */
2076
0
        if(target>=targetLimit) {
2077
            /* target is full */
2078
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2079
0
            break;
2080
0
        }
2081
2082
0
        entry=stateTable[0][*source++];
2083
        /* MBCS_ENTRY_IS_FINAL(entry) */
2084
2085
        /* test the most common case first */
2086
0
        if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
2087
            /* output BMP code point */
2088
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2089
0
            if(offsets!=NULL) {
2090
0
                *offsets++=sourceIndex;
2091
0
            }
2092
2093
            /* normal end of action codes: prepare for a new character */
2094
0
            ++sourceIndex;
2095
0
            continue;
2096
0
        }
2097
2098
        /*
2099
         * An if-else-if chain provides more reliable performance for
2100
         * the most common cases compared to a switch.
2101
         */
2102
0
        action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
2103
0
        if(action==MBCS_STATE_VALID_DIRECT_20 ||
2104
0
           (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
2105
0
        ) {
2106
0
            entry=MBCS_ENTRY_FINAL_VALUE(entry);
2107
            /* output surrogate pair */
2108
0
            *target++=(UChar)(0xd800|(UChar)(entry>>10));
2109
0
            if(offsets!=NULL) {
2110
0
                *offsets++=sourceIndex;
2111
0
            }
2112
0
            c=(UChar)(0xdc00|(UChar)(entry&0x3ff));
2113
0
            if(target<targetLimit) {
2114
0
                *target++=c;
2115
0
                if(offsets!=NULL) {
2116
0
                    *offsets++=sourceIndex;
2117
0
                }
2118
0
            } else {
2119
                /* target overflow */
2120
0
                cnv->UCharErrorBuffer[0]=c;
2121
0
                cnv->UCharErrorBufferLength=1;
2122
0
                *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2123
0
                break;
2124
0
            }
2125
2126
0
            ++sourceIndex;
2127
0
            continue;
2128
0
        } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
2129
0
            if(UCNV_TO_U_USE_FALLBACK(cnv)) {
2130
                /* output BMP code point */
2131
0
                *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2132
0
                if(offsets!=NULL) {
2133
0
                    *offsets++=sourceIndex;
2134
0
                }
2135
2136
0
                ++sourceIndex;
2137
0
                continue;
2138
0
            }
2139
0
        } else if(action==MBCS_STATE_UNASSIGNED) {
2140
            /* just fall through */
2141
0
        } else if(action==MBCS_STATE_ILLEGAL) {
2142
            /* callback(illegal) */
2143
0
            *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2144
0
        } else {
2145
            /* reserved, must never occur */
2146
0
            ++sourceIndex;
2147
0
            continue;
2148
0
        }
2149
2150
0
        if(U_FAILURE(*pErrorCode)) {
2151
            /* callback(illegal) */
2152
0
            break;
2153
0
        } else /* unassigned sequences indicated with byteIndex>0 */ {
2154
            /* try an extension mapping */
2155
0
            pArgs->source=(const char *)source;
2156
0
            cnv->toUBytes[0]=*(source-1);
2157
0
            cnv->toULength=_extToU(cnv, cnv->sharedData,
2158
0
                                    1, &source, sourceLimit,
2159
0
                                    &target, targetLimit,
2160
0
                                    &offsets, sourceIndex,
2161
0
                                    pArgs->flush,
2162
0
                                    pErrorCode);
2163
0
            sourceIndex+=1+(int32_t)(source-(const uint8_t *)pArgs->source);
2164
2165
0
            if(U_FAILURE(*pErrorCode)) {
2166
                /* not mappable or buffer overflow */
2167
0
                break;
2168
0
            }
2169
0
        }
2170
0
    }
2171
2172
    /* write back the updated pointers */
2173
0
    pArgs->source=(const char *)source;
2174
0
    pArgs->target=target;
2175
0
    pArgs->offsets=offsets;
2176
0
}
2177
2178
/*
2179
 * This version of ucnv_MBCSSingleToUnicodeWithOffsets() is optimized for single-byte, single-state codepages
2180
 * that only map to and from the BMP.
2181
 * In addition to single-byte optimizations, the offset calculations
2182
 * become much easier.
2183
 */
2184
static void
2185
ucnv_MBCSSingleToBMPWithOffsets(UConverterToUnicodeArgs *pArgs,
2186
0
                            UErrorCode *pErrorCode) {
2187
0
    UConverter *cnv;
2188
0
    const uint8_t *source, *sourceLimit, *lastSource;
2189
0
    UChar *target;
2190
0
    int32_t targetCapacity, length;
2191
0
    int32_t *offsets;
2192
2193
0
    const int32_t (*stateTable)[256];
2194
2195
0
    int32_t sourceIndex;
2196
2197
0
    int32_t entry;
2198
0
    uint8_t action;
2199
2200
    /* set up the local pointers */
2201
0
    cnv=pArgs->converter;
2202
0
    source=(const uint8_t *)pArgs->source;
2203
0
    sourceLimit=(const uint8_t *)pArgs->sourceLimit;
2204
0
    target=pArgs->target;
2205
0
    targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target);
2206
0
    offsets=pArgs->offsets;
2207
2208
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
2209
0
        stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
2210
0
    } else {
2211
0
        stateTable=cnv->sharedData->mbcs.stateTable;
2212
0
    }
2213
2214
    /* sourceIndex=-1 if the current character began in the previous buffer */
2215
0
    sourceIndex=0;
2216
0
    lastSource=source;
2217
2218
    /*
2219
     * since the conversion here is 1:1 UChar:uint8_t, we need only one counter
2220
     * for the minimum of the sourceLength and targetCapacity
2221
     */
2222
0
    length=(int32_t)(sourceLimit-source);
2223
0
    if(length<targetCapacity) {
2224
0
        targetCapacity=length;
2225
0
    }
2226
2227
0
#if MBCS_UNROLL_SINGLE_TO_BMP
2228
    /* unrolling makes it faster on Pentium III/Windows 2000 */
2229
    /* unroll the loop with the most common case */
2230
0
unrolled:
2231
0
    if(targetCapacity>=16) {
2232
0
        int32_t count, loops, oredEntries;
2233
2234
0
        loops=count=targetCapacity>>4;
2235
0
        do {
2236
0
            oredEntries=entry=stateTable[0][*source++];
2237
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2238
0
            oredEntries|=entry=stateTable[0][*source++];
2239
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2240
0
            oredEntries|=entry=stateTable[0][*source++];
2241
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2242
0
            oredEntries|=entry=stateTable[0][*source++];
2243
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2244
0
            oredEntries|=entry=stateTable[0][*source++];
2245
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2246
0
            oredEntries|=entry=stateTable[0][*source++];
2247
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2248
0
            oredEntries|=entry=stateTable[0][*source++];
2249
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2250
0
            oredEntries|=entry=stateTable[0][*source++];
2251
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2252
0
            oredEntries|=entry=stateTable[0][*source++];
2253
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2254
0
            oredEntries|=entry=stateTable[0][*source++];
2255
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2256
0
            oredEntries|=entry=stateTable[0][*source++];
2257
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2258
0
            oredEntries|=entry=stateTable[0][*source++];
2259
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2260
0
            oredEntries|=entry=stateTable[0][*source++];
2261
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2262
0
            oredEntries|=entry=stateTable[0][*source++];
2263
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2264
0
            oredEntries|=entry=stateTable[0][*source++];
2265
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2266
0
            oredEntries|=entry=stateTable[0][*source++];
2267
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2268
2269
            /* were all 16 entries really valid? */
2270
0
            if(!MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(oredEntries)) {
2271
                /* no, return to the first of these 16 */
2272
0
                source-=16;
2273
0
                target-=16;
2274
0
                break;
2275
0
            }
2276
0
        } while(--count>0);
2277
0
        count=loops-count;
2278
0
        targetCapacity-=16*count;
2279
2280
0
        if(offsets!=NULL) {
2281
0
            lastSource+=16*count;
2282
0
            while(count>0) {
2283
0
                *offsets++=sourceIndex++;
2284
0
                *offsets++=sourceIndex++;
2285
0
                *offsets++=sourceIndex++;
2286
0
                *offsets++=sourceIndex++;
2287
0
                *offsets++=sourceIndex++;
2288
0
                *offsets++=sourceIndex++;
2289
0
                *offsets++=sourceIndex++;
2290
0
                *offsets++=sourceIndex++;
2291
0
                *offsets++=sourceIndex++;
2292
0
                *offsets++=sourceIndex++;
2293
0
                *offsets++=sourceIndex++;
2294
0
                *offsets++=sourceIndex++;
2295
0
                *offsets++=sourceIndex++;
2296
0
                *offsets++=sourceIndex++;
2297
0
                *offsets++=sourceIndex++;
2298
0
                *offsets++=sourceIndex++;
2299
0
                --count;
2300
0
            }
2301
0
        }
2302
0
    }
2303
0
#endif
2304
2305
    /* conversion loop */
2306
0
    while(targetCapacity > 0 && source < sourceLimit) {
2307
0
        entry=stateTable[0][*source++];
2308
        /* MBCS_ENTRY_IS_FINAL(entry) */
2309
2310
        /* test the most common case first */
2311
0
        if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
2312
            /* output BMP code point */
2313
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2314
0
            --targetCapacity;
2315
0
            continue;
2316
0
        }
2317
2318
        /*
2319
         * An if-else-if chain provides more reliable performance for
2320
         * the most common cases compared to a switch.
2321
         */
2322
0
        action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
2323
0
        if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
2324
0
            if(UCNV_TO_U_USE_FALLBACK(cnv)) {
2325
                /* output BMP code point */
2326
0
                *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2327
0
                --targetCapacity;
2328
0
                continue;
2329
0
            }
2330
0
        } else if(action==MBCS_STATE_UNASSIGNED) {
2331
            /* just fall through */
2332
0
        } else if(action==MBCS_STATE_ILLEGAL) {
2333
            /* callback(illegal) */
2334
0
            *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2335
0
        } else {
2336
            /* reserved, must never occur */
2337
0
            continue;
2338
0
        }
2339
2340
        /* set offsets since the start or the last extension */
2341
0
        if(offsets!=NULL) {
2342
0
            int32_t count=(int32_t)(source-lastSource);
2343
2344
            /* predecrement: do not set the offset for the callback-causing character */
2345
0
            while(--count>0) {
2346
0
                *offsets++=sourceIndex++;
2347
0
            }
2348
            /* offset and sourceIndex are now set for the current character */
2349
0
        }
2350
2351
0
        if(U_FAILURE(*pErrorCode)) {
2352
            /* callback(illegal) */
2353
0
            break;
2354
0
        } else /* unassigned sequences indicated with byteIndex>0 */ {
2355
            /* try an extension mapping */
2356
0
            lastSource=source;
2357
0
            cnv->toUBytes[0]=*(source-1);
2358
0
            cnv->toULength=_extToU(cnv, cnv->sharedData,
2359
0
                                    1, &source, sourceLimit,
2360
0
                                    &target, pArgs->targetLimit,
2361
0
                                    &offsets, sourceIndex,
2362
0
                                    pArgs->flush,
2363
0
                                    pErrorCode);
2364
0
            sourceIndex+=1+(int32_t)(source-lastSource);
2365
2366
0
            if(U_FAILURE(*pErrorCode)) {
2367
                /* not mappable or buffer overflow */
2368
0
                break;
2369
0
            }
2370
2371
            /* recalculate the targetCapacity after an extension mapping */
2372
0
            targetCapacity=(int32_t)(pArgs->targetLimit-target);
2373
0
            length=(int32_t)(sourceLimit-source);
2374
0
            if(length<targetCapacity) {
2375
0
                targetCapacity=length;
2376
0
            }
2377
0
        }
2378
2379
0
#if MBCS_UNROLL_SINGLE_TO_BMP
2380
        /* unrolling makes it faster on Pentium III/Windows 2000 */
2381
0
        goto unrolled;
2382
0
#endif
2383
0
    }
2384
2385
0
    if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=pArgs->targetLimit) {
2386
        /* target is full */
2387
0
        *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2388
0
    }
2389
2390
    /* set offsets since the start or the last callback */
2391
0
    if(offsets!=NULL) {
2392
0
        size_t count=source-lastSource;
2393
0
        while(count>0) {
2394
0
            *offsets++=sourceIndex++;
2395
0
            --count;
2396
0
        }
2397
0
    }
2398
2399
    /* write back the updated pointers */
2400
0
    pArgs->source=(const char *)source;
2401
0
    pArgs->target=target;
2402
0
    pArgs->offsets=offsets;
2403
0
}
2404
2405
static UBool
2406
0
hasValidTrailBytes(const int32_t (*stateTable)[256], uint8_t state) {
2407
0
    const int32_t *row=stateTable[state];
2408
0
    int32_t b, entry;
2409
    /* First test for final entries in this state for some commonly valid byte values. */
2410
0
    entry=row[0xa1];
2411
0
    if( !MBCS_ENTRY_IS_TRANSITION(entry) &&
2412
0
        MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL
2413
0
    ) {
2414
0
        return TRUE;
2415
0
    }
2416
0
    entry=row[0x41];
2417
0
    if( !MBCS_ENTRY_IS_TRANSITION(entry) &&
2418
0
        MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL
2419
0
    ) {
2420
0
        return TRUE;
2421
0
    }
2422
    /* Then test for final entries in this state. */
2423
0
    for(b=0; b<=0xff; ++b) {
2424
0
        entry=row[b];
2425
0
        if( !MBCS_ENTRY_IS_TRANSITION(entry) &&
2426
0
            MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL
2427
0
        ) {
2428
0
            return TRUE;
2429
0
        }
2430
0
    }
2431
    /* Then recurse for transition entries. */
2432
0
    for(b=0; b<=0xff; ++b) {
2433
0
        entry=row[b];
2434
0
        if( MBCS_ENTRY_IS_TRANSITION(entry) &&
2435
0
            hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry))
2436
0
        ) {
2437
0
            return TRUE;
2438
0
        }
2439
0
    }
2440
0
    return FALSE;
2441
0
}
2442
2443
/*
2444
 * Is byte b a single/lead byte in this state?
2445
 * Recurse for transition states, because here we don't want to say that
2446
 * b is a lead byte if all byte sequences that start with b are illegal.
2447
 */
2448
static UBool
2449
0
isSingleOrLead(const int32_t (*stateTable)[256], uint8_t state, UBool isDBCSOnly, uint8_t b) {
2450
0
    const int32_t *row=stateTable[state];
2451
0
    int32_t entry=row[b];
2452
0
    if(MBCS_ENTRY_IS_TRANSITION(entry)) {   /* lead byte */
2453
0
        return hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry));
2454
0
    } else {
2455
0
        uint8_t action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
2456
0
        if(action==MBCS_STATE_CHANGE_ONLY && isDBCSOnly) {
2457
0
            return FALSE;   /* SI/SO are illegal for DBCS-only conversion */
2458
0
        } else {
2459
0
            return action!=MBCS_STATE_ILLEGAL;
2460
0
        }
2461
0
    }
2462
0
}
2463
2464
U_CFUNC void
2465
ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
2466
0
                          UErrorCode *pErrorCode) {
2467
0
    UConverter *cnv;
2468
0
    const uint8_t *source, *sourceLimit;
2469
0
    UChar *target;
2470
0
    const UChar *targetLimit;
2471
0
    int32_t *offsets;
2472
2473
0
    const int32_t (*stateTable)[256];
2474
0
    const uint16_t *unicodeCodeUnits;
2475
2476
0
    uint32_t offset;
2477
0
    uint8_t state;
2478
0
    int8_t byteIndex;
2479
0
    uint8_t *bytes;
2480
2481
0
    int32_t sourceIndex, nextSourceIndex;
2482
2483
0
    int32_t entry;
2484
0
    UChar c;
2485
0
    uint8_t action;
2486
2487
    /* use optimized function if possible */
2488
0
    cnv=pArgs->converter;
2489
2490
0
    if(cnv->preToULength>0) {
2491
        /*
2492
         * pass sourceIndex=-1 because we continue from an earlier buffer
2493
         * in the future, this may change with continuous offsets
2494
         */
2495
0
        ucnv_extContinueMatchToU(cnv, pArgs, -1, pErrorCode);
2496
2497
0
        if(U_FAILURE(*pErrorCode) || cnv->preToULength<0) {
2498
0
            return;
2499
0
        }
2500
0
    }
2501
2502
0
    if(cnv->sharedData->mbcs.countStates==1) {
2503
0
        if(!(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
2504
0
            ucnv_MBCSSingleToBMPWithOffsets(pArgs, pErrorCode);
2505
0
        } else {
2506
0
            ucnv_MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode);
2507
0
        }
2508
0
        return;
2509
0
    }
2510
2511
    /* set up the local pointers */
2512
0
    source=(const uint8_t *)pArgs->source;
2513
0
    sourceLimit=(const uint8_t *)pArgs->sourceLimit;
2514
0
    target=pArgs->target;
2515
0
    targetLimit=pArgs->targetLimit;
2516
0
    offsets=pArgs->offsets;
2517
2518
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
2519
0
        stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
2520
0
    } else {
2521
0
        stateTable=cnv->sharedData->mbcs.stateTable;
2522
0
    }
2523
0
    unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits;
2524
2525
    /* get the converter state from UConverter */
2526
0
    offset=cnv->toUnicodeStatus;
2527
0
    byteIndex=cnv->toULength;
2528
0
    bytes=cnv->toUBytes;
2529
2530
    /*
2531
     * if we are in the SBCS state for a DBCS-only converter,
2532
     * then load the DBCS state from the MBCS data
2533
     * (dbcsOnlyState==0 if it is not a DBCS-only converter)
2534
     */
2535
0
    if((state=(uint8_t)(cnv->mode))==0) {
2536
0
        state=cnv->sharedData->mbcs.dbcsOnlyState;
2537
0
    }
2538
2539
    /* sourceIndex=-1 if the current character began in the previous buffer */
2540
0
    sourceIndex=byteIndex==0 ? 0 : -1;
2541
0
    nextSourceIndex=0;
2542
2543
    /* conversion loop */
2544
0
    while(source<sourceLimit) {
2545
        /*
2546
         * This following test is to see if available input would overflow the output.
2547
         * It does not catch output of more than one code unit that
2548
         * overflows as a result of a surrogate pair or callback output
2549
         * from the last source byte.
2550
         * Therefore, those situations also test for overflows and will
2551
         * then break the loop, too.
2552
         */
2553
0
        if(target>=targetLimit) {
2554
            /* target is full */
2555
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2556
0
            break;
2557
0
        }
2558
2559
0
        if(byteIndex==0) {
2560
            /* optimized loop for 1/2-byte input and BMP output */
2561
0
            if(offsets==NULL) {
2562
0
                do {
2563
0
                    entry=stateTable[state][*source];
2564
0
                    if(MBCS_ENTRY_IS_TRANSITION(entry)) {
2565
0
                        state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
2566
0
                        offset=MBCS_ENTRY_TRANSITION_OFFSET(entry);
2567
2568
0
                        ++source;
2569
0
                        if( source<sourceLimit &&
2570
0
                            MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) &&
2571
0
                            MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 &&
2572
0
                            (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe
2573
0
                        ) {
2574
0
                            ++source;
2575
0
                            *target++=c;
2576
0
                            state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
2577
0
                            offset=0;
2578
0
                        } else {
2579
                            /* set the state and leave the optimized loop */
2580
0
                            bytes[0]=*(source-1);
2581
0
                            byteIndex=1;
2582
0
                            break;
2583
0
                        }
2584
0
                    } else {
2585
0
                        if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
2586
                            /* output BMP code point */
2587
0
                            ++source;
2588
0
                            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2589
0
                            state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
2590
0
                        } else {
2591
                            /* leave the optimized loop */
2592
0
                            break;
2593
0
                        }
2594
0
                    }
2595
0
                } while(source<sourceLimit && target<targetLimit);
2596
0
            } else /* offsets!=NULL */ {
2597
0
                do {
2598
0
                    entry=stateTable[state][*source];
2599
0
                    if(MBCS_ENTRY_IS_TRANSITION(entry)) {
2600
0
                        state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
2601
0
                        offset=MBCS_ENTRY_TRANSITION_OFFSET(entry);
2602
2603
0
                        ++source;
2604
0
                        if( source<sourceLimit &&
2605
0
                            MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) &&
2606
0
                            MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 &&
2607
0
                            (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe
2608
0
                        ) {
2609
0
                            ++source;
2610
0
                            *target++=c;
2611
0
                            if(offsets!=NULL) {
2612
0
                                *offsets++=sourceIndex;
2613
0
                                sourceIndex=(nextSourceIndex+=2);
2614
0
                            }
2615
0
                            state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
2616
0
                            offset=0;
2617
0
                        } else {
2618
                            /* set the state and leave the optimized loop */
2619
0
                            ++nextSourceIndex;
2620
0
                            bytes[0]=*(source-1);
2621
0
                            byteIndex=1;
2622
0
                            break;
2623
0
                        }
2624
0
                    } else {
2625
0
                        if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
2626
                            /* output BMP code point */
2627
0
                            ++source;
2628
0
                            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2629
0
                            if(offsets!=NULL) {
2630
0
                                *offsets++=sourceIndex;
2631
0
                                sourceIndex=++nextSourceIndex;
2632
0
                            }
2633
0
                            state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
2634
0
                        } else {
2635
                            /* leave the optimized loop */
2636
0
                            break;
2637
0
                        }
2638
0
                    }
2639
0
                } while(source<sourceLimit && target<targetLimit);
2640
0
            }
2641
2642
            /*
2643
             * these tests and break statements could be put inside the loop
2644
             * if C had "break outerLoop" like Java
2645
             */
2646
0
            if(source>=sourceLimit) {
2647
0
                break;
2648
0
            }
2649
0
            if(target>=targetLimit) {
2650
                /* target is full */
2651
0
                *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2652
0
                break;
2653
0
            }
2654
2655
0
            ++nextSourceIndex;
2656
0
            bytes[byteIndex++]=*source++;
2657
0
        } else /* byteIndex>0 */ {
2658
0
            ++nextSourceIndex;
2659
0
            entry=stateTable[state][bytes[byteIndex++]=*source++];
2660
0
        }
2661
2662
0
        if(MBCS_ENTRY_IS_TRANSITION(entry)) {
2663
0
            state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
2664
0
            offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
2665
0
            continue;
2666
0
        }
2667
2668
        /* save the previous state for proper extension mapping with SI/SO-stateful converters */
2669
0
        cnv->mode=state;
2670
2671
        /* set the next state early so that we can reuse the entry variable */
2672
0
        state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
2673
2674
        /*
2675
         * An if-else-if chain provides more reliable performance for
2676
         * the most common cases compared to a switch.
2677
         */
2678
0
        action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
2679
0
        if(action==MBCS_STATE_VALID_16) {
2680
0
            offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
2681
0
            c=unicodeCodeUnits[offset];
2682
0
            if(c<0xfffe) {
2683
                /* output BMP code point */
2684
0
                *target++=c;
2685
0
                if(offsets!=NULL) {
2686
0
                    *offsets++=sourceIndex;
2687
0
                }
2688
0
                byteIndex=0;
2689
0
            } else if(c==0xfffe) {
2690
0
                if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) {
2691
                    /* output fallback BMP code point */
2692
0
                    *target++=(UChar)entry;
2693
0
                    if(offsets!=NULL) {
2694
0
                        *offsets++=sourceIndex;
2695
0
                    }
2696
0
                    byteIndex=0;
2697
0
                }
2698
0
            } else {
2699
                /* callback(illegal) */
2700
0
                *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2701
0
            }
2702
0
        } else if(action==MBCS_STATE_VALID_DIRECT_16) {
2703
            /* output BMP code point */
2704
0
            *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2705
0
            if(offsets!=NULL) {
2706
0
                *offsets++=sourceIndex;
2707
0
            }
2708
0
            byteIndex=0;
2709
0
        } else if(action==MBCS_STATE_VALID_16_PAIR) {
2710
0
            offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
2711
0
            c=unicodeCodeUnits[offset++];
2712
0
            if(c<0xd800) {
2713
                /* output BMP code point below 0xd800 */
2714
0
                *target++=c;
2715
0
                if(offsets!=NULL) {
2716
0
                    *offsets++=sourceIndex;
2717
0
                }
2718
0
                byteIndex=0;
2719
0
            } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) {
2720
                /* output roundtrip or fallback surrogate pair */
2721
0
                *target++=(UChar)(c&0xdbff);
2722
0
                if(offsets!=NULL) {
2723
0
                    *offsets++=sourceIndex;
2724
0
                }
2725
0
                byteIndex=0;
2726
0
                if(target<targetLimit) {
2727
0
                    *target++=unicodeCodeUnits[offset];
2728
0
                    if(offsets!=NULL) {
2729
0
                        *offsets++=sourceIndex;
2730
0
                    }
2731
0
                } else {
2732
                    /* target overflow */
2733
0
                    cnv->UCharErrorBuffer[0]=unicodeCodeUnits[offset];
2734
0
                    cnv->UCharErrorBufferLength=1;
2735
0
                    *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2736
2737
0
                    offset=0;
2738
0
                    break;
2739
0
                }
2740
0
            } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) {
2741
                /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
2742
0
                *target++=unicodeCodeUnits[offset];
2743
0
                if(offsets!=NULL) {
2744
0
                    *offsets++=sourceIndex;
2745
0
                }
2746
0
                byteIndex=0;
2747
0
            } else if(c==0xffff) {
2748
                /* callback(illegal) */
2749
0
                *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2750
0
            }
2751
0
        } else if(action==MBCS_STATE_VALID_DIRECT_20 ||
2752
0
                  (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
2753
0
        ) {
2754
0
            entry=MBCS_ENTRY_FINAL_VALUE(entry);
2755
            /* output surrogate pair */
2756
0
            *target++=(UChar)(0xd800|(UChar)(entry>>10));
2757
0
            if(offsets!=NULL) {
2758
0
                *offsets++=sourceIndex;
2759
0
            }
2760
0
            byteIndex=0;
2761
0
            c=(UChar)(0xdc00|(UChar)(entry&0x3ff));
2762
0
            if(target<targetLimit) {
2763
0
                *target++=c;
2764
0
                if(offsets!=NULL) {
2765
0
                    *offsets++=sourceIndex;
2766
0
                }
2767
0
            } else {
2768
                /* target overflow */
2769
0
                cnv->UCharErrorBuffer[0]=c;
2770
0
                cnv->UCharErrorBufferLength=1;
2771
0
                *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
2772
2773
0
                offset=0;
2774
0
                break;
2775
0
            }
2776
0
        } else if(action==MBCS_STATE_CHANGE_ONLY) {
2777
            /*
2778
             * This serves as a state change without any output.
2779
             * It is useful for reading simple stateful encodings,
2780
             * for example using just Shift-In/Shift-Out codes.
2781
             * The 21 unused bits may later be used for more sophisticated
2782
             * state transitions.
2783
             */
2784
0
            if(cnv->sharedData->mbcs.dbcsOnlyState==0) {
2785
0
                byteIndex=0;
2786
0
            } else {
2787
                /* SI/SO are illegal for DBCS-only conversion */
2788
0
                state=(uint8_t)(cnv->mode); /* restore the previous state */
2789
2790
                /* callback(illegal) */
2791
0
                *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2792
0
            }
2793
0
        } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
2794
0
            if(UCNV_TO_U_USE_FALLBACK(cnv)) {
2795
                /* output BMP code point */
2796
0
                *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2797
0
                if(offsets!=NULL) {
2798
0
                    *offsets++=sourceIndex;
2799
0
                }
2800
0
                byteIndex=0;
2801
0
            }
2802
0
        } else if(action==MBCS_STATE_UNASSIGNED) {
2803
            /* just fall through */
2804
0
        } else if(action==MBCS_STATE_ILLEGAL) {
2805
            /* callback(illegal) */
2806
0
            *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2807
0
        } else {
2808
            /* reserved, must never occur */
2809
0
            byteIndex=0;
2810
0
        }
2811
2812
        /* end of action codes: prepare for a new character */
2813
0
        offset=0;
2814
2815
0
        if(byteIndex==0) {
2816
0
            sourceIndex=nextSourceIndex;
2817
0
        } else if(U_FAILURE(*pErrorCode)) {
2818
            /* callback(illegal) */
2819
0
            if(byteIndex>1) {
2820
                /*
2821
                 * Ticket 5691: consistent illegal sequences:
2822
                 * - We include at least the first byte in the illegal sequence.
2823
                 * - If any of the non-initial bytes could be the start of a character,
2824
                 *   we stop the illegal sequence before the first one of those.
2825
                 */
2826
0
                UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0);
2827
0
                int8_t i;
2828
0
                for(i=1;
2829
0
                    i<byteIndex && !isSingleOrLead(stateTable, state, isDBCSOnly, bytes[i]);
2830
0
                    ++i) {}
2831
0
                if(i<byteIndex) {
2832
                    /* Back out some bytes. */
2833
0
                    int8_t backOutDistance=byteIndex-i;
2834
0
                    int32_t bytesFromThisBuffer=(int32_t)(source-(const uint8_t *)pArgs->source);
2835
0
                    byteIndex=i;  /* length of reported illegal byte sequence */
2836
0
                    if(backOutDistance<=bytesFromThisBuffer) {
2837
0
                        source-=backOutDistance;
2838
0
                    } else {
2839
                        /* Back out bytes from the previous buffer: Need to replay them. */
2840
0
                        cnv->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance);
2841
                        /* preToULength is negative! */
2842
0
                        uprv_memcpy(cnv->preToU, bytes+i, -cnv->preToULength);
2843
0
                        source=(const uint8_t *)pArgs->source;
2844
0
                    }
2845
0
                }
2846
0
            }
2847
0
            break;
2848
0
        } else /* unassigned sequences indicated with byteIndex>0 */ {
2849
            /* try an extension mapping */
2850
0
            pArgs->source=(const char *)source;
2851
0
            byteIndex=_extToU(cnv, cnv->sharedData,
2852
0
                              byteIndex, &source, sourceLimit,
2853
0
                              &target, targetLimit,
2854
0
                              &offsets, sourceIndex,
2855
0
                              pArgs->flush,
2856
0
                              pErrorCode);
2857
0
            sourceIndex=nextSourceIndex+=(int32_t)(source-(const uint8_t *)pArgs->source);
2858
2859
0
            if(U_FAILURE(*pErrorCode)) {
2860
                /* not mappable or buffer overflow */
2861
0
                break;
2862
0
            }
2863
0
        }
2864
0
    }
2865
2866
    /* set the converter state back into UConverter */
2867
0
    cnv->toUnicodeStatus=offset;
2868
0
    cnv->mode=state;
2869
0
    cnv->toULength=byteIndex;
2870
2871
    /* write back the updated pointers */
2872
0
    pArgs->source=(const char *)source;
2873
0
    pArgs->target=target;
2874
0
    pArgs->offsets=offsets;
2875
0
}
2876
2877
/*
2878
 * This version of ucnv_MBCSGetNextUChar() is optimized for single-byte, single-state codepages.
2879
 * We still need a conversion loop in case we find reserved action codes, which are to be ignored.
2880
 */
2881
static UChar32
2882
ucnv_MBCSSingleGetNextUChar(UConverterToUnicodeArgs *pArgs,
2883
0
                        UErrorCode *pErrorCode) {
2884
0
    UConverter *cnv;
2885
0
    const int32_t (*stateTable)[256];
2886
0
    const uint8_t *source, *sourceLimit;
2887
2888
0
    int32_t entry;
2889
0
    uint8_t action;
2890
2891
    /* set up the local pointers */
2892
0
    cnv=pArgs->converter;
2893
0
    source=(const uint8_t *)pArgs->source;
2894
0
    sourceLimit=(const uint8_t *)pArgs->sourceLimit;
2895
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
2896
0
        stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
2897
0
    } else {
2898
0
        stateTable=cnv->sharedData->mbcs.stateTable;
2899
0
    }
2900
2901
    /* conversion loop */
2902
0
    while(source<sourceLimit) {
2903
0
        entry=stateTable[0][*source++];
2904
        /* MBCS_ENTRY_IS_FINAL(entry) */
2905
2906
        /* write back the updated pointer early so that we can return directly */
2907
0
        pArgs->source=(const char *)source;
2908
2909
0
        if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
2910
            /* output BMP code point */
2911
0
            return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2912
0
        }
2913
2914
        /*
2915
         * An if-else-if chain provides more reliable performance for
2916
         * the most common cases compared to a switch.
2917
         */
2918
0
        action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
2919
0
        if( action==MBCS_STATE_VALID_DIRECT_20 ||
2920
0
            (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
2921
0
        ) {
2922
            /* output supplementary code point */
2923
0
            return (UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000);
2924
0
        } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
2925
0
            if(UCNV_TO_U_USE_FALLBACK(cnv)) {
2926
                /* output BMP code point */
2927
0
                return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
2928
0
            }
2929
0
        } else if(action==MBCS_STATE_UNASSIGNED) {
2930
            /* just fall through */
2931
0
        } else if(action==MBCS_STATE_ILLEGAL) {
2932
            /* callback(illegal) */
2933
0
            *pErrorCode=U_ILLEGAL_CHAR_FOUND;
2934
0
        } else {
2935
            /* reserved, must never occur */
2936
0
            continue;
2937
0
        }
2938
2939
0
        if(U_FAILURE(*pErrorCode)) {
2940
            /* callback(illegal) */
2941
0
            break;
2942
0
        } else /* unassigned sequence */ {
2943
            /* defer to the generic implementation */
2944
0
            pArgs->source=(const char *)source-1;
2945
0
            return UCNV_GET_NEXT_UCHAR_USE_TO_U;
2946
0
        }
2947
0
    }
2948
2949
    /* no output because of empty input or only state changes */
2950
0
    *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
2951
0
    return 0xffff;
2952
0
}
2953
2954
/*
2955
 * Version of _MBCSToUnicodeWithOffsets() optimized for single-character
2956
 * conversion without offset handling.
2957
 *
2958
 * When a character does not have a mapping to Unicode, then we return to the
2959
 * generic ucnv_getNextUChar() code for extension/GB 18030 and error/callback
2960
 * handling.
2961
 * We also defer to the generic code in other complicated cases and have them
2962
 * ultimately handled by _MBCSToUnicodeWithOffsets() itself.
2963
 *
2964
 * All normal mappings and errors are handled here.
2965
 */
2966
static UChar32 U_CALLCONV
2967
ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs,
2968
0
                  UErrorCode *pErrorCode) {
2969
0
    UConverter *cnv;
2970
0
    const uint8_t *source, *sourceLimit, *lastSource;
2971
2972
0
    const int32_t (*stateTable)[256];
2973
0
    const uint16_t *unicodeCodeUnits;
2974
2975
0
    uint32_t offset;
2976
0
    uint8_t state;
2977
2978
0
    int32_t entry;
2979
0
    UChar32 c;
2980
0
    uint8_t action;
2981
2982
    /* use optimized function if possible */
2983
0
    cnv=pArgs->converter;
2984
2985
0
    if(cnv->preToULength>0) {
2986
        /* use the generic code in ucnv_getNextUChar() to continue with a partial match */
2987
0
        return UCNV_GET_NEXT_UCHAR_USE_TO_U;
2988
0
    }
2989
2990
0
    if(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SURROGATES) {
2991
        /*
2992
         * Using the generic ucnv_getNextUChar() code lets us deal correctly
2993
         * with the rare case of a codepage that maps single surrogates
2994
         * without adding the complexity to this already complicated function here.
2995
         */
2996
0
        return UCNV_GET_NEXT_UCHAR_USE_TO_U;
2997
0
    } else if(cnv->sharedData->mbcs.countStates==1) {
2998
0
        return ucnv_MBCSSingleGetNextUChar(pArgs, pErrorCode);
2999
0
    }
3000
3001
    /* set up the local pointers */
3002
0
    source=lastSource=(const uint8_t *)pArgs->source;
3003
0
    sourceLimit=(const uint8_t *)pArgs->sourceLimit;
3004
3005
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
3006
0
        stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
3007
0
    } else {
3008
0
        stateTable=cnv->sharedData->mbcs.stateTable;
3009
0
    }
3010
0
    unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits;
3011
3012
    /* get the converter state from UConverter */
3013
0
    offset=cnv->toUnicodeStatus;
3014
3015
    /*
3016
     * if we are in the SBCS state for a DBCS-only converter,
3017
     * then load the DBCS state from the MBCS data
3018
     * (dbcsOnlyState==0 if it is not a DBCS-only converter)
3019
     */
3020
0
    if((state=(uint8_t)(cnv->mode))==0) {
3021
0
        state=cnv->sharedData->mbcs.dbcsOnlyState;
3022
0
    }
3023
3024
    /* conversion loop */
3025
0
    c=U_SENTINEL;
3026
0
    while(source<sourceLimit) {
3027
0
        entry=stateTable[state][*source++];
3028
0
        if(MBCS_ENTRY_IS_TRANSITION(entry)) {
3029
0
            state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
3030
0
            offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
3031
3032
            /* optimization for 1/2-byte input and BMP output */
3033
0
            if( source<sourceLimit &&
3034
0
                MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) &&
3035
0
                MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 &&
3036
0
                (c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe
3037
0
            ) {
3038
0
                ++source;
3039
0
                state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
3040
                /* output BMP code point */
3041
0
                break;
3042
0
            }
3043
0
        } else {
3044
            /* save the previous state for proper extension mapping with SI/SO-stateful converters */
3045
0
            cnv->mode=state;
3046
3047
            /* set the next state early so that we can reuse the entry variable */
3048
0
            state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
3049
3050
            /*
3051
             * An if-else-if chain provides more reliable performance for
3052
             * the most common cases compared to a switch.
3053
             */
3054
0
            action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
3055
0
            if(action==MBCS_STATE_VALID_DIRECT_16) {
3056
                /* output BMP code point */
3057
0
                c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
3058
0
                break;
3059
0
            } else if(action==MBCS_STATE_VALID_16) {
3060
0
                offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
3061
0
                c=unicodeCodeUnits[offset];
3062
0
                if(c<0xfffe) {
3063
                    /* output BMP code point */
3064
0
                    break;
3065
0
                } else if(c==0xfffe) {
3066
0
                    if(UCNV_TO_U_USE_FALLBACK(cnv) && (c=ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) {
3067
0
                        break;
3068
0
                    }
3069
0
                } else {
3070
                    /* callback(illegal) */
3071
0
                    *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3072
0
                }
3073
0
            } else if(action==MBCS_STATE_VALID_16_PAIR) {
3074
0
                offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
3075
0
                c=unicodeCodeUnits[offset++];
3076
0
                if(c<0xd800) {
3077
                    /* output BMP code point below 0xd800 */
3078
0
                    break;
3079
0
                } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) {
3080
                    /* output roundtrip or fallback supplementary code point */
3081
0
                    c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00);
3082
0
                    break;
3083
0
                } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) {
3084
                    /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
3085
0
                    c=unicodeCodeUnits[offset];
3086
0
                    break;
3087
0
                } else if(c==0xffff) {
3088
                    /* callback(illegal) */
3089
0
                    *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3090
0
                }
3091
0
            } else if(action==MBCS_STATE_VALID_DIRECT_20 ||
3092
0
                      (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
3093
0
            ) {
3094
                /* output supplementary code point */
3095
0
                c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000);
3096
0
                break;
3097
0
            } else if(action==MBCS_STATE_CHANGE_ONLY) {
3098
                /*
3099
                 * This serves as a state change without any output.
3100
                 * It is useful for reading simple stateful encodings,
3101
                 * for example using just Shift-In/Shift-Out codes.
3102
                 * The 21 unused bits may later be used for more sophisticated
3103
                 * state transitions.
3104
                 */
3105
0
                if(cnv->sharedData->mbcs.dbcsOnlyState!=0) {
3106
                    /* SI/SO are illegal for DBCS-only conversion */
3107
0
                    state=(uint8_t)(cnv->mode); /* restore the previous state */
3108
3109
                    /* callback(illegal) */
3110
0
                    *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3111
0
                }
3112
0
            } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
3113
0
                if(UCNV_TO_U_USE_FALLBACK(cnv)) {
3114
                    /* output BMP code point */
3115
0
                    c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
3116
0
                    break;
3117
0
                }
3118
0
            } else if(action==MBCS_STATE_UNASSIGNED) {
3119
                /* just fall through */
3120
0
            } else if(action==MBCS_STATE_ILLEGAL) {
3121
                /* callback(illegal) */
3122
0
                *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3123
0
            } else {
3124
                /* reserved (must never occur), or only state change */
3125
0
                offset=0;
3126
0
                lastSource=source;
3127
0
                continue;
3128
0
            }
3129
3130
            /* end of action codes: prepare for a new character */
3131
0
            offset=0;
3132
3133
0
            if(U_FAILURE(*pErrorCode)) {
3134
                /* callback(illegal) */
3135
0
                break;
3136
0
            } else /* unassigned sequence */ {
3137
                /* defer to the generic implementation */
3138
0
                cnv->toUnicodeStatus=0;
3139
0
                cnv->mode=state;
3140
0
                pArgs->source=(const char *)lastSource;
3141
0
                return UCNV_GET_NEXT_UCHAR_USE_TO_U;
3142
0
            }
3143
0
        }
3144
0
    }
3145
3146
0
    if(c<0) {
3147
0
        if(U_SUCCESS(*pErrorCode) && source==sourceLimit && lastSource<source) {
3148
            /* incomplete character byte sequence */
3149
0
            uint8_t *bytes=cnv->toUBytes;
3150
0
            cnv->toULength=(int8_t)(source-lastSource);
3151
0
            do {
3152
0
                *bytes++=*lastSource++;
3153
0
            } while(lastSource<source);
3154
0
            *pErrorCode=U_TRUNCATED_CHAR_FOUND;
3155
0
        } else if(U_FAILURE(*pErrorCode)) {
3156
            /* callback(illegal) */
3157
            /*
3158
             * Ticket 5691: consistent illegal sequences:
3159
             * - We include at least the first byte in the illegal sequence.
3160
             * - If any of the non-initial bytes could be the start of a character,
3161
             *   we stop the illegal sequence before the first one of those.
3162
             */
3163
0
            UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0);
3164
0
            uint8_t *bytes=cnv->toUBytes;
3165
0
            *bytes++=*lastSource++;     /* first byte */
3166
0
            if(lastSource==source) {
3167
0
                cnv->toULength=1;
3168
0
            } else /* lastSource<source: multi-byte character */ {
3169
0
                int8_t i;
3170
0
                for(i=1;
3171
0
                    lastSource<source && !isSingleOrLead(stateTable, state, isDBCSOnly, *lastSource);
3172
0
                    ++i
3173
0
                ) {
3174
0
                    *bytes++=*lastSource++;
3175
0
                }
3176
0
                cnv->toULength=i;
3177
0
                source=lastSource;
3178
0
            }
3179
0
        } else {
3180
            /* no output because of empty input or only state changes */
3181
0
            *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
3182
0
        }
3183
0
        c=0xffff;
3184
0
    }
3185
3186
    /* set the converter state back into UConverter, ready for a new character */
3187
0
    cnv->toUnicodeStatus=0;
3188
0
    cnv->mode=state;
3189
3190
    /* write back the updated pointer */
3191
0
    pArgs->source=(const char *)source;
3192
0
    return c;
3193
0
}
3194
3195
#if 0
3196
/*
3197
 * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus
3198
 * Removal improves code coverage.
3199
 */
3200
/**
3201
 * This version of ucnv_MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
3202
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
3203
 * It does not handle conversion extensions (_extToU()).
3204
 */
3205
U_CFUNC UChar32
3206
ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
3207
                              uint8_t b, UBool useFallback) {
3208
    int32_t entry;
3209
    uint8_t action;
3210
3211
    entry=sharedData->mbcs.stateTable[0][b];
3212
    /* MBCS_ENTRY_IS_FINAL(entry) */
3213
3214
    if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
3215
        /* output BMP code point */
3216
        return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
3217
    }
3218
3219
    /*
3220
     * An if-else-if chain provides more reliable performance for
3221
     * the most common cases compared to a switch.
3222
     */
3223
    action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
3224
    if(action==MBCS_STATE_VALID_DIRECT_20) {
3225
        /* output supplementary code point */
3226
        return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry);
3227
    } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
3228
        if(!TO_U_USE_FALLBACK(useFallback)) {
3229
            return 0xfffe;
3230
        }
3231
        /* output BMP code point */
3232
        return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
3233
    } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) {
3234
        if(!TO_U_USE_FALLBACK(useFallback)) {
3235
            return 0xfffe;
3236
        }
3237
        /* output supplementary code point */
3238
        return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry);
3239
    } else if(action==MBCS_STATE_UNASSIGNED) {
3240
        return 0xfffe;
3241
    } else if(action==MBCS_STATE_ILLEGAL) {
3242
        return 0xffff;
3243
    } else {
3244
        /* reserved, must never occur */
3245
        return 0xffff;
3246
    }
3247
}
3248
#endif
3249
3250
/*
3251
 * This is a simple version of _MBCSGetNextUChar() that is used
3252
 * by other converter implementations.
3253
 * It only returns an "assigned" result if it consumes the entire input.
3254
 * It does not use state from the converter, nor error codes.
3255
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
3256
 * It handles conversion extensions but not GB 18030.
3257
 *
3258
 * Return value:
3259
 * U+fffe   unassigned
3260
 * U+ffff   illegal
3261
 * otherwise the Unicode code point
3262
 */
3263
U_CFUNC UChar32
3264
ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
3265
                        const char *source, int32_t length,
3266
0
                        UBool useFallback) {
3267
0
    const int32_t (*stateTable)[256];
3268
0
    const uint16_t *unicodeCodeUnits;
3269
3270
0
    uint32_t offset;
3271
0
    uint8_t state, action;
3272
3273
0
    UChar32 c;
3274
0
    int32_t i, entry;
3275
3276
0
    if(length<=0) {
3277
        /* no input at all: "illegal" */
3278
0
        return 0xffff;
3279
0
    }
3280
3281
#if 0
3282
/*
3283
 * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus
3284
 * TODO In future releases, verify that this function is never called for SBCS
3285
 * conversions, i.e., that sharedData->mbcs.countStates==1 is still true.
3286
 * Removal improves code coverage.
3287
 */
3288
    /* use optimized function if possible */
3289
    if(sharedData->mbcs.countStates==1) {
3290
        if(length==1) {
3291
            return ucnv_MBCSSingleSimpleGetNextUChar(sharedData, (uint8_t)*source, useFallback);
3292
        } else {
3293
            return 0xffff; /* illegal: more than a single byte for an SBCS converter */
3294
        }
3295
    }
3296
#endif
3297
3298
    /* set up the local pointers */
3299
0
    stateTable=sharedData->mbcs.stateTable;
3300
0
    unicodeCodeUnits=sharedData->mbcs.unicodeCodeUnits;
3301
3302
    /* converter state */
3303
0
    offset=0;
3304
0
    state=sharedData->mbcs.dbcsOnlyState;
3305
3306
    /* conversion loop */
3307
0
    for(i=0;;) {
3308
0
        entry=stateTable[state][(uint8_t)source[i++]];
3309
0
        if(MBCS_ENTRY_IS_TRANSITION(entry)) {
3310
0
            state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
3311
0
            offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
3312
3313
0
            if(i==length) {
3314
0
                return 0xffff; /* truncated character */
3315
0
            }
3316
0
        } else {
3317
            /*
3318
             * An if-else-if chain provides more reliable performance for
3319
             * the most common cases compared to a switch.
3320
             */
3321
0
            action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
3322
0
            if(action==MBCS_STATE_VALID_16) {
3323
0
                offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
3324
0
                c=unicodeCodeUnits[offset];
3325
0
                if(c!=0xfffe) {
3326
                    /* done */
3327
0
                } else if(UCNV_TO_U_USE_FALLBACK(cnv)) {
3328
0
                    c=ucnv_MBCSGetFallback(&sharedData->mbcs, offset);
3329
                /* else done with 0xfffe */
3330
0
                }
3331
0
                break;
3332
0
            } else if(action==MBCS_STATE_VALID_DIRECT_16) {
3333
                /* output BMP code point */
3334
0
                c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
3335
0
                break;
3336
0
            } else if(action==MBCS_STATE_VALID_16_PAIR) {
3337
0
                offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
3338
0
                c=unicodeCodeUnits[offset++];
3339
0
                if(c<0xd800) {
3340
                    /* output BMP code point below 0xd800 */
3341
0
                } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) {
3342
                    /* output roundtrip or fallback supplementary code point */
3343
0
                    c=(UChar32)(((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00));
3344
0
                } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) {
3345
                    /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
3346
0
                    c=unicodeCodeUnits[offset];
3347
0
                } else if(c==0xffff) {
3348
0
                    return 0xffff;
3349
0
                } else {
3350
0
                    c=0xfffe;
3351
0
                }
3352
0
                break;
3353
0
            } else if(action==MBCS_STATE_VALID_DIRECT_20) {
3354
                /* output supplementary code point */
3355
0
                c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry);
3356
0
                break;
3357
0
            } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
3358
0
                if(!TO_U_USE_FALLBACK(useFallback)) {
3359
0
                    c=0xfffe;
3360
0
                    break;
3361
0
                }
3362
                /* output BMP code point */
3363
0
                c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
3364
0
                break;
3365
0
            } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) {
3366
0
                if(!TO_U_USE_FALLBACK(useFallback)) {
3367
0
                    c=0xfffe;
3368
0
                    break;
3369
0
                }
3370
                /* output supplementary code point */
3371
0
                c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry);
3372
0
                break;
3373
0
            } else if(action==MBCS_STATE_UNASSIGNED) {
3374
0
                c=0xfffe;
3375
0
                break;
3376
0
            }
3377
3378
            /*
3379
             * forbid MBCS_STATE_CHANGE_ONLY for this function,
3380
             * and MBCS_STATE_ILLEGAL and reserved action codes
3381
             */
3382
0
            return 0xffff;
3383
0
        }
3384
0
    }
3385
3386
0
    if(i!=length) {
3387
        /* illegal for this function: not all input consumed */
3388
0
        return 0xffff;
3389
0
    }
3390
3391
0
    if(c==0xfffe) {
3392
        /* try an extension mapping */
3393
0
        const int32_t *cx=sharedData->mbcs.extIndexes;
3394
0
        if(cx!=NULL) {
3395
0
            return ucnv_extSimpleMatchToU(cx, source, length, useFallback);
3396
0
        }
3397
0
    }
3398
3399
0
    return c;
3400
0
}
3401
3402
/* MBCS-from-Unicode conversion functions ----------------------------------- */
3403
3404
/* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for double-byte codepages. */
3405
static void
3406
ucnv_MBCSDoubleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
3407
0
                                  UErrorCode *pErrorCode) {
3408
0
    UConverter *cnv;
3409
0
    const UChar *source, *sourceLimit;
3410
0
    uint8_t *target;
3411
0
    int32_t targetCapacity;
3412
0
    int32_t *offsets;
3413
3414
0
    const uint16_t *table;
3415
0
    const uint16_t *mbcsIndex;
3416
0
    const uint8_t *bytes;
3417
3418
0
    UChar32 c;
3419
3420
0
    int32_t sourceIndex, nextSourceIndex;
3421
3422
0
    uint32_t stage2Entry;
3423
0
    uint32_t asciiRoundtrips;
3424
0
    uint32_t value;
3425
0
    uint8_t unicodeMask;
3426
3427
    /* use optimized function if possible */
3428
0
    cnv=pArgs->converter;
3429
0
    unicodeMask=cnv->sharedData->mbcs.unicodeMask;
3430
3431
    /* set up the local pointers */
3432
0
    source=pArgs->source;
3433
0
    sourceLimit=pArgs->sourceLimit;
3434
0
    target=(uint8_t *)pArgs->target;
3435
0
    targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target);
3436
0
    offsets=pArgs->offsets;
3437
3438
0
    table=cnv->sharedData->mbcs.fromUnicodeTable;
3439
0
    mbcsIndex=cnv->sharedData->mbcs.mbcsIndex;
3440
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
3441
0
        bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes;
3442
0
    } else {
3443
0
        bytes=cnv->sharedData->mbcs.fromUnicodeBytes;
3444
0
    }
3445
0
    asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips;
3446
3447
    /* get the converter state from UConverter */
3448
0
    c=cnv->fromUChar32;
3449
3450
    /* sourceIndex=-1 if the current character began in the previous buffer */
3451
0
    sourceIndex= c==0 ? 0 : -1;
3452
0
    nextSourceIndex=0;
3453
3454
    /* conversion loop */
3455
0
    if(c!=0 && targetCapacity>0) {
3456
0
        goto getTrail;
3457
0
    }
3458
3459
0
    while(source<sourceLimit) {
3460
        /*
3461
         * This following test is to see if available input would overflow the output.
3462
         * It does not catch output of more than one byte that
3463
         * overflows as a result of a multi-byte character or callback output
3464
         * from the last source character.
3465
         * Therefore, those situations also test for overflows and will
3466
         * then break the loop, too.
3467
         */
3468
0
        if(targetCapacity>0) {
3469
            /*
3470
             * Get a correct Unicode code point:
3471
             * a single UChar for a BMP code point or
3472
             * a matched surrogate pair for a "supplementary code point".
3473
             */
3474
0
            c=*source++;
3475
0
            ++nextSourceIndex;
3476
0
            if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) {
3477
0
                *target++=(uint8_t)c;
3478
0
                if(offsets!=NULL) {
3479
0
                    *offsets++=sourceIndex;
3480
0
                    sourceIndex=nextSourceIndex;
3481
0
                }
3482
0
                --targetCapacity;
3483
0
                c=0;
3484
0
                continue;
3485
0
            }
3486
            /*
3487
             * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX
3488
             * to avoid dealing with surrogates.
3489
             * MBCS_FAST_MAX must be >=0xd7ff.
3490
             */
3491
0
            if(c<=0xd7ff) {
3492
0
                value=DBCS_RESULT_FROM_MOST_BMP(mbcsIndex, (const uint16_t *)bytes, c);
3493
                /* There are only roundtrips (!=0) and no-mapping (==0) entries. */
3494
0
                if(value==0) {
3495
0
                    goto unassigned;
3496
0
                }
3497
                /* output the value */
3498
0
            } else {
3499
                /*
3500
                 * This also tests if the codepage maps single surrogates.
3501
                 * If it does, then surrogates are not paired but mapped separately.
3502
                 * Note that in this case unmatched surrogates are not detected.
3503
                 */
3504
0
                if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) {
3505
0
                    if(U16_IS_SURROGATE_LEAD(c)) {
3506
0
getTrail:
3507
0
                        if(source<sourceLimit) {
3508
                            /* test the following code unit */
3509
0
                            UChar trail=*source;
3510
0
                            if(U16_IS_TRAIL(trail)) {
3511
0
                                ++source;
3512
0
                                ++nextSourceIndex;
3513
0
                                c=U16_GET_SUPPLEMENTARY(c, trail);
3514
0
                                if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
3515
                                    /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
3516
                                    /* callback(unassigned) */
3517
0
                                    goto unassigned;
3518
0
                                }
3519
                                /* convert this supplementary code point */
3520
                                /* exit this condition tree */
3521
0
                            } else {
3522
                                /* this is an unmatched lead code unit (1st surrogate) */
3523
                                /* callback(illegal) */
3524
0
                                *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3525
0
                                break;
3526
0
                            }
3527
0
                        } else {
3528
                            /* no more input */
3529
0
                            break;
3530
0
                        }
3531
0
                    } else {
3532
                        /* this is an unmatched trail code unit (2nd surrogate) */
3533
                        /* callback(illegal) */
3534
0
                        *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3535
0
                        break;
3536
0
                    }
3537
0
                }
3538
3539
                /* convert the Unicode code point in c into codepage bytes */
3540
0
                stage2Entry=MBCS_STAGE_2_FROM_U(table, c);
3541
3542
                /* get the bytes and the length for the output */
3543
                /* MBCS_OUTPUT_2 */
3544
0
                value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c);
3545
3546
                /* is this code point assigned, or do we use fallbacks? */
3547
0
                if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ||
3548
0
                     (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0))
3549
0
                ) {
3550
                    /*
3551
                     * We allow a 0 byte output if the "assigned" bit is set for this entry.
3552
                     * There is no way with this data structure for fallback output
3553
                     * to be a zero byte.
3554
                     */
3555
3556
0
unassigned:
3557
                    /* try an extension mapping */
3558
0
                    pArgs->source=source;
3559
0
                    c=_extFromU(cnv, cnv->sharedData,
3560
0
                                c, &source, sourceLimit,
3561
0
                                &target, target+targetCapacity,
3562
0
                                &offsets, sourceIndex,
3563
0
                                pArgs->flush,
3564
0
                                pErrorCode);
3565
0
                    nextSourceIndex+=(int32_t)(source-pArgs->source);
3566
3567
0
                    if(U_FAILURE(*pErrorCode)) {
3568
                        /* not mappable or buffer overflow */
3569
0
                        break;
3570
0
                    } else {
3571
                        /* a mapping was written to the target, continue */
3572
3573
                        /* recalculate the targetCapacity after an extension mapping */
3574
0
                        targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target);
3575
3576
                        /* normal end of conversion: prepare for a new character */
3577
0
                        sourceIndex=nextSourceIndex;
3578
0
                        continue;
3579
0
                    }
3580
0
                }
3581
0
            }
3582
3583
            /* write the output character bytes from value and length */
3584
            /* from the first if in the loop we know that targetCapacity>0 */
3585
0
            if(value<=0xff) {
3586
                /* this is easy because we know that there is enough space */
3587
0
                *target++=(uint8_t)value;
3588
0
                if(offsets!=NULL) {
3589
0
                    *offsets++=sourceIndex;
3590
0
                }
3591
0
                --targetCapacity;
3592
0
            } else /* length==2 */ {
3593
0
                *target++=(uint8_t)(value>>8);
3594
0
                if(2<=targetCapacity) {
3595
0
                    *target++=(uint8_t)value;
3596
0
                    if(offsets!=NULL) {
3597
0
                        *offsets++=sourceIndex;
3598
0
                        *offsets++=sourceIndex;
3599
0
                    }
3600
0
                    targetCapacity-=2;
3601
0
                } else {
3602
0
                    if(offsets!=NULL) {
3603
0
                        *offsets++=sourceIndex;
3604
0
                    }
3605
0
                    cnv->charErrorBuffer[0]=(char)value;
3606
0
                    cnv->charErrorBufferLength=1;
3607
3608
                    /* target overflow */
3609
0
                    targetCapacity=0;
3610
0
                    *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
3611
0
                    c=0;
3612
0
                    break;
3613
0
                }
3614
0
            }
3615
3616
            /* normal end of conversion: prepare for a new character */
3617
0
            c=0;
3618
0
            sourceIndex=nextSourceIndex;
3619
0
            continue;
3620
0
        } else {
3621
            /* target is full */
3622
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
3623
0
            break;
3624
0
        }
3625
0
    }
3626
3627
    /* set the converter state back into UConverter */
3628
0
    cnv->fromUChar32=c;
3629
3630
    /* write back the updated pointers */
3631
0
    pArgs->source=source;
3632
0
    pArgs->target=(char *)target;
3633
0
    pArgs->offsets=offsets;
3634
0
}
3635
3636
/* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for single-byte codepages. */
3637
static void
3638
ucnv_MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
3639
0
                                  UErrorCode *pErrorCode) {
3640
0
    UConverter *cnv;
3641
0
    const UChar *source, *sourceLimit;
3642
0
    uint8_t *target;
3643
0
    int32_t targetCapacity;
3644
0
    int32_t *offsets;
3645
3646
0
    const uint16_t *table;
3647
0
    const uint16_t *results;
3648
3649
0
    UChar32 c;
3650
3651
0
    int32_t sourceIndex, nextSourceIndex;
3652
3653
0
    uint16_t value, minValue;
3654
0
    UBool hasSupplementary;
3655
3656
    /* set up the local pointers */
3657
0
    cnv=pArgs->converter;
3658
0
    source=pArgs->source;
3659
0
    sourceLimit=pArgs->sourceLimit;
3660
0
    target=(uint8_t *)pArgs->target;
3661
0
    targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target);
3662
0
    offsets=pArgs->offsets;
3663
3664
0
    table=cnv->sharedData->mbcs.fromUnicodeTable;
3665
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
3666
0
        results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes;
3667
0
    } else {
3668
0
        results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes;
3669
0
    }
3670
3671
0
    if(cnv->useFallback) {
3672
        /* use all roundtrip and fallback results */
3673
0
        minValue=0x800;
3674
0
    } else {
3675
        /* use only roundtrips and fallbacks from private-use characters */
3676
0
        minValue=0xc00;
3677
0
    }
3678
0
    hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY);
3679
3680
    /* get the converter state from UConverter */
3681
0
    c=cnv->fromUChar32;
3682
3683
    /* sourceIndex=-1 if the current character began in the previous buffer */
3684
0
    sourceIndex= c==0 ? 0 : -1;
3685
0
    nextSourceIndex=0;
3686
3687
    /* conversion loop */
3688
0
    if(c!=0 && targetCapacity>0) {
3689
0
        goto getTrail;
3690
0
    }
3691
3692
0
    while(source<sourceLimit) {
3693
        /*
3694
         * This following test is to see if available input would overflow the output.
3695
         * It does not catch output of more than one byte that
3696
         * overflows as a result of a multi-byte character or callback output
3697
         * from the last source character.
3698
         * Therefore, those situations also test for overflows and will
3699
         * then break the loop, too.
3700
         */
3701
0
        if(targetCapacity>0) {
3702
            /*
3703
             * Get a correct Unicode code point:
3704
             * a single UChar for a BMP code point or
3705
             * a matched surrogate pair for a "supplementary code point".
3706
             */
3707
0
            c=*source++;
3708
0
            ++nextSourceIndex;
3709
0
            if(U16_IS_SURROGATE(c)) {
3710
0
                if(U16_IS_SURROGATE_LEAD(c)) {
3711
0
getTrail:
3712
0
                    if(source<sourceLimit) {
3713
                        /* test the following code unit */
3714
0
                        UChar trail=*source;
3715
0
                        if(U16_IS_TRAIL(trail)) {
3716
0
                            ++source;
3717
0
                            ++nextSourceIndex;
3718
0
                            c=U16_GET_SUPPLEMENTARY(c, trail);
3719
0
                            if(!hasSupplementary) {
3720
                                /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
3721
                                /* callback(unassigned) */
3722
0
                                goto unassigned;
3723
0
                            }
3724
                            /* convert this supplementary code point */
3725
                            /* exit this condition tree */
3726
0
                        } else {
3727
                            /* this is an unmatched lead code unit (1st surrogate) */
3728
                            /* callback(illegal) */
3729
0
                            *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3730
0
                            break;
3731
0
                        }
3732
0
                    } else {
3733
                        /* no more input */
3734
0
                        break;
3735
0
                    }
3736
0
                } else {
3737
                    /* this is an unmatched trail code unit (2nd surrogate) */
3738
                    /* callback(illegal) */
3739
0
                    *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3740
0
                    break;
3741
0
                }
3742
0
            }
3743
3744
            /* convert the Unicode code point in c into codepage bytes */
3745
0
            value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
3746
3747
            /* is this code point assigned, or do we use fallbacks? */
3748
0
            if(value>=minValue) {
3749
                /* assigned, write the output character bytes from value and length */
3750
                /* length==1 */
3751
                /* this is easy because we know that there is enough space */
3752
0
                *target++=(uint8_t)value;
3753
0
                if(offsets!=NULL) {
3754
0
                    *offsets++=sourceIndex;
3755
0
                }
3756
0
                --targetCapacity;
3757
3758
                /* normal end of conversion: prepare for a new character */
3759
0
                c=0;
3760
0
                sourceIndex=nextSourceIndex;
3761
0
            } else { /* unassigned */
3762
0
unassigned:
3763
                /* try an extension mapping */
3764
0
                pArgs->source=source;
3765
0
                c=_extFromU(cnv, cnv->sharedData,
3766
0
                            c, &source, sourceLimit,
3767
0
                            &target, target+targetCapacity,
3768
0
                            &offsets, sourceIndex,
3769
0
                            pArgs->flush,
3770
0
                            pErrorCode);
3771
0
                nextSourceIndex+=(int32_t)(source-pArgs->source);
3772
3773
0
                if(U_FAILURE(*pErrorCode)) {
3774
                    /* not mappable or buffer overflow */
3775
0
                    break;
3776
0
                } else {
3777
                    /* a mapping was written to the target, continue */
3778
3779
                    /* recalculate the targetCapacity after an extension mapping */
3780
0
                    targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target);
3781
3782
                    /* normal end of conversion: prepare for a new character */
3783
0
                    sourceIndex=nextSourceIndex;
3784
0
                }
3785
0
            }
3786
0
        } else {
3787
            /* target is full */
3788
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
3789
0
            break;
3790
0
        }
3791
0
    }
3792
3793
    /* set the converter state back into UConverter */
3794
0
    cnv->fromUChar32=c;
3795
3796
    /* write back the updated pointers */
3797
0
    pArgs->source=source;
3798
0
    pArgs->target=(char *)target;
3799
0
    pArgs->offsets=offsets;
3800
0
}
3801
3802
/*
3803
 * This version of ucnv_MBCSFromUnicode() is optimized for single-byte codepages
3804
 * that map only to and from the BMP.
3805
 * In addition to single-byte/state optimizations, the offset calculations
3806
 * become much easier.
3807
 * It would be possible to use the sbcsIndex for UTF-8-friendly tables,
3808
 * but measurements have shown that this diminishes performance
3809
 * in more cases than it improves it.
3810
 * See SVN revision 21013 (2007-feb-06) for the last version with #if switches
3811
 * for various MBCS and SBCS optimizations.
3812
 */
3813
static void
3814
ucnv_MBCSSingleFromBMPWithOffsets(UConverterFromUnicodeArgs *pArgs,
3815
0
                              UErrorCode *pErrorCode) {
3816
0
    UConverter *cnv;
3817
0
    const UChar *source, *sourceLimit, *lastSource;
3818
0
    uint8_t *target;
3819
0
    int32_t targetCapacity, length;
3820
0
    int32_t *offsets;
3821
3822
0
    const uint16_t *table;
3823
0
    const uint16_t *results;
3824
3825
0
    UChar32 c;
3826
3827
0
    int32_t sourceIndex;
3828
3829
0
    uint32_t asciiRoundtrips;
3830
0
    uint16_t value, minValue;
3831
3832
    /* set up the local pointers */
3833
0
    cnv=pArgs->converter;
3834
0
    source=pArgs->source;
3835
0
    sourceLimit=pArgs->sourceLimit;
3836
0
    target=(uint8_t *)pArgs->target;
3837
0
    targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target);
3838
0
    offsets=pArgs->offsets;
3839
3840
0
    table=cnv->sharedData->mbcs.fromUnicodeTable;
3841
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
3842
0
        results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes;
3843
0
    } else {
3844
0
        results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes;
3845
0
    }
3846
0
    asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips;
3847
3848
0
    if(cnv->useFallback) {
3849
        /* use all roundtrip and fallback results */
3850
0
        minValue=0x800;
3851
0
    } else {
3852
        /* use only roundtrips and fallbacks from private-use characters */
3853
0
        minValue=0xc00;
3854
0
    }
3855
3856
    /* get the converter state from UConverter */
3857
0
    c=cnv->fromUChar32;
3858
3859
    /* sourceIndex=-1 if the current character began in the previous buffer */
3860
0
    sourceIndex= c==0 ? 0 : -1;
3861
0
    lastSource=source;
3862
3863
    /*
3864
     * since the conversion here is 1:1 UChar:uint8_t, we need only one counter
3865
     * for the minimum of the sourceLength and targetCapacity
3866
     */
3867
0
    length=(int32_t)(sourceLimit-source);
3868
0
    if(length<targetCapacity) {
3869
0
        targetCapacity=length;
3870
0
    }
3871
3872
    /* conversion loop */
3873
0
    if(c!=0 && targetCapacity>0) {
3874
0
        goto getTrail;
3875
0
    }
3876
3877
#if MBCS_UNROLL_SINGLE_FROM_BMP
3878
    /* unrolling makes it slower on Pentium III/Windows 2000?! */
3879
    /* unroll the loop with the most common case */
3880
unrolled:
3881
    if(targetCapacity>=4) {
3882
        int32_t count, loops;
3883
        uint16_t andedValues;
3884
3885
        loops=count=targetCapacity>>2;
3886
        do {
3887
            c=*source++;
3888
            andedValues=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
3889
            *target++=(uint8_t)value;
3890
            c=*source++;
3891
            andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
3892
            *target++=(uint8_t)value;
3893
            c=*source++;
3894
            andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
3895
            *target++=(uint8_t)value;
3896
            c=*source++;
3897
            andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
3898
            *target++=(uint8_t)value;
3899
3900
            /* were all 4 entries really valid? */
3901
            if(andedValues<minValue) {
3902
                /* no, return to the first of these 4 */
3903
                source-=4;
3904
                target-=4;
3905
                break;
3906
            }
3907
        } while(--count>0);
3908
        count=loops-count;
3909
        targetCapacity-=4*count;
3910
3911
        if(offsets!=NULL) {
3912
            lastSource+=4*count;
3913
            while(count>0) {
3914
                *offsets++=sourceIndex++;
3915
                *offsets++=sourceIndex++;
3916
                *offsets++=sourceIndex++;
3917
                *offsets++=sourceIndex++;
3918
                --count;
3919
            }
3920
        }
3921
3922
        c=0;
3923
    }
3924
#endif
3925
3926
0
    while(targetCapacity>0) {
3927
        /*
3928
         * Get a correct Unicode code point:
3929
         * a single UChar for a BMP code point or
3930
         * a matched surrogate pair for a "supplementary code point".
3931
         */
3932
0
        c=*source++;
3933
        /*
3934
         * Do not immediately check for single surrogates:
3935
         * Assume that they are unassigned and check for them in that case.
3936
         * This speeds up the conversion of assigned characters.
3937
         */
3938
        /* convert the Unicode code point in c into codepage bytes */
3939
0
        if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) {
3940
0
            *target++=(uint8_t)c;
3941
0
            --targetCapacity;
3942
0
            c=0;
3943
0
            continue;
3944
0
        }
3945
0
        value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
3946
        /* is this code point assigned, or do we use fallbacks? */
3947
0
        if(value>=minValue) {
3948
            /* assigned, write the output character bytes from value and length */
3949
            /* length==1 */
3950
            /* this is easy because we know that there is enough space */
3951
0
            *target++=(uint8_t)value;
3952
0
            --targetCapacity;
3953
3954
            /* normal end of conversion: prepare for a new character */
3955
0
            c=0;
3956
0
            continue;
3957
0
        } else if(!U16_IS_SURROGATE(c)) {
3958
            /* normal, unassigned BMP character */
3959
0
        } else if(U16_IS_SURROGATE_LEAD(c)) {
3960
0
getTrail:
3961
0
            if(source<sourceLimit) {
3962
                /* test the following code unit */
3963
0
                UChar trail=*source;
3964
0
                if(U16_IS_TRAIL(trail)) {
3965
0
                    ++source;
3966
0
                    c=U16_GET_SUPPLEMENTARY(c, trail);
3967
                    /* this codepage does not map supplementary code points */
3968
                    /* callback(unassigned) */
3969
0
                } else {
3970
                    /* this is an unmatched lead code unit (1st surrogate) */
3971
                    /* callback(illegal) */
3972
0
                    *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3973
0
                    break;
3974
0
                }
3975
0
            } else {
3976
                /* no more input */
3977
0
                if (pArgs->flush) {
3978
0
                    *pErrorCode=U_TRUNCATED_CHAR_FOUND;
3979
0
                }
3980
0
                break;
3981
0
            }
3982
0
        } else {
3983
            /* this is an unmatched trail code unit (2nd surrogate) */
3984
            /* callback(illegal) */
3985
0
            *pErrorCode=U_ILLEGAL_CHAR_FOUND;
3986
0
            break;
3987
0
        }
3988
3989
        /* c does not have a mapping */
3990
3991
        /* get the number of code units for c to correctly advance sourceIndex */
3992
0
        length=U16_LENGTH(c);
3993
3994
        /* set offsets since the start or the last extension */
3995
0
        if(offsets!=NULL) {
3996
0
            int32_t count=(int32_t)(source-lastSource);
3997
3998
            /* do not set the offset for this character */
3999
0
            count-=length;
4000
4001
0
            while(count>0) {
4002
0
                *offsets++=sourceIndex++;
4003
0
                --count;
4004
0
            }
4005
            /* offsets and sourceIndex are now set for the current character */
4006
0
        }
4007
4008
        /* try an extension mapping */
4009
0
        lastSource=source;
4010
0
        c=_extFromU(cnv, cnv->sharedData,
4011
0
                    c, &source, sourceLimit,
4012
0
                    &target, (const uint8_t *)(pArgs->targetLimit),
4013
0
                    &offsets, sourceIndex,
4014
0
                    pArgs->flush,
4015
0
                    pErrorCode);
4016
0
        sourceIndex+=length+(int32_t)(source-lastSource);
4017
0
        lastSource=source;
4018
4019
0
        if(U_FAILURE(*pErrorCode)) {
4020
            /* not mappable or buffer overflow */
4021
0
            break;
4022
0
        } else {
4023
            /* a mapping was written to the target, continue */
4024
4025
            /* recalculate the targetCapacity after an extension mapping */
4026
0
            targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target);
4027
0
            length=(int32_t)(sourceLimit-source);
4028
0
            if(length<targetCapacity) {
4029
0
                targetCapacity=length;
4030
0
            }
4031
0
        }
4032
4033
#if MBCS_UNROLL_SINGLE_FROM_BMP
4034
        /* unrolling makes it slower on Pentium III/Windows 2000?! */
4035
        goto unrolled;
4036
#endif
4037
0
    }
4038
4039
0
    if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=(uint8_t *)pArgs->targetLimit) {
4040
        /* target is full */
4041
0
        *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
4042
0
    }
4043
4044
    /* set offsets since the start or the last callback */
4045
0
    if(offsets!=NULL) {
4046
0
        size_t count=source-lastSource;
4047
0
        if (count > 0 && *pErrorCode == U_TRUNCATED_CHAR_FOUND) {
4048
            /*
4049
            Caller gave us a partial supplementary character,
4050
            which this function couldn't convert in any case.
4051
            The callback will handle the offset.
4052
            */
4053
0
            count--;
4054
0
        }
4055
0
        while(count>0) {
4056
0
            *offsets++=sourceIndex++;
4057
0
            --count;
4058
0
        }
4059
0
    }
4060
4061
    /* set the converter state back into UConverter */
4062
0
    cnv->fromUChar32=c;
4063
4064
    /* write back the updated pointers */
4065
0
    pArgs->source=source;
4066
0
    pArgs->target=(char *)target;
4067
0
    pArgs->offsets=offsets;
4068
0
}
4069
4070
U_CFUNC void
4071
ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
4072
0
                            UErrorCode *pErrorCode) {
4073
0
    UConverter *cnv;
4074
0
    const UChar *source, *sourceLimit;
4075
0
    uint8_t *target;
4076
0
    int32_t targetCapacity;
4077
0
    int32_t *offsets;
4078
4079
0
    const uint16_t *table;
4080
0
    const uint16_t *mbcsIndex;
4081
0
    const uint8_t *p, *bytes;
4082
0
    uint8_t outputType;
4083
4084
0
    UChar32 c;
4085
4086
0
    int32_t prevSourceIndex, sourceIndex, nextSourceIndex;
4087
4088
0
    uint32_t stage2Entry;
4089
0
    uint32_t asciiRoundtrips;
4090
0
    uint32_t value;
4091
    /* Shift-In and Shift-Out byte sequences differ by encoding scheme. */
4092
0
    uint8_t siBytes[2] = {0, 0};
4093
0
    uint8_t soBytes[2] = {0, 0};
4094
0
    uint8_t siLength, soLength;
4095
0
    int32_t length = 0, prevLength;
4096
0
    uint8_t unicodeMask;
4097
4098
0
    cnv=pArgs->converter;
4099
4100
0
    if(cnv->preFromUFirstCP>=0) {
4101
        /*
4102
         * pass sourceIndex=-1 because we continue from an earlier buffer
4103
         * in the future, this may change with continuous offsets
4104
         */
4105
0
        ucnv_extContinueMatchFromU(cnv, pArgs, -1, pErrorCode);
4106
4107
0
        if(U_FAILURE(*pErrorCode) || cnv->preFromULength<0) {
4108
0
            return;
4109
0
        }
4110
0
    }
4111
4112
    /* use optimized function if possible */
4113
0
    outputType=cnv->sharedData->mbcs.outputType;
4114
0
    unicodeMask=cnv->sharedData->mbcs.unicodeMask;
4115
0
    if(outputType==MBCS_OUTPUT_1 && !(unicodeMask&UCNV_HAS_SURROGATES)) {
4116
0
        if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
4117
0
            ucnv_MBCSSingleFromBMPWithOffsets(pArgs, pErrorCode);
4118
0
        } else {
4119
0
            ucnv_MBCSSingleFromUnicodeWithOffsets(pArgs, pErrorCode);
4120
0
        }
4121
0
        return;
4122
0
    } else if(outputType==MBCS_OUTPUT_2 && cnv->sharedData->mbcs.utf8Friendly) {
4123
0
        ucnv_MBCSDoubleFromUnicodeWithOffsets(pArgs, pErrorCode);
4124
0
        return;
4125
0
    }
4126
4127
    /* set up the local pointers */
4128
0
    source=pArgs->source;
4129
0
    sourceLimit=pArgs->sourceLimit;
4130
0
    target=(uint8_t *)pArgs->target;
4131
0
    targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target);
4132
0
    offsets=pArgs->offsets;
4133
4134
0
    table=cnv->sharedData->mbcs.fromUnicodeTable;
4135
0
    if(cnv->sharedData->mbcs.utf8Friendly) {
4136
0
        mbcsIndex=cnv->sharedData->mbcs.mbcsIndex;
4137
0
    } else {
4138
0
        mbcsIndex=NULL;
4139
0
    }
4140
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
4141
0
        bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes;
4142
0
    } else {
4143
0
        bytes=cnv->sharedData->mbcs.fromUnicodeBytes;
4144
0
    }
4145
0
    asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips;
4146
4147
    /* get the converter state from UConverter */
4148
0
    c=cnv->fromUChar32;
4149
4150
0
    if(outputType==MBCS_OUTPUT_2_SISO) {
4151
0
        prevLength=cnv->fromUnicodeStatus;
4152
0
        if(prevLength==0) {
4153
            /* set the real value */
4154
0
            prevLength=1;
4155
0
        }
4156
0
    } else {
4157
        /* prevent fromUnicodeStatus from being set to something non-0 */
4158
0
        prevLength=0;
4159
0
    }
4160
4161
    /* sourceIndex=-1 if the current character began in the previous buffer */
4162
0
    prevSourceIndex=-1;
4163
0
    sourceIndex= c==0 ? 0 : -1;
4164
0
    nextSourceIndex=0;
4165
4166
    /* Get the SI/SO character for the converter */
4167
0
    siLength = static_cast<uint8_t>(getSISOBytes(SI, cnv->options, siBytes));
4168
0
    soLength = static_cast<uint8_t>(getSISOBytes(SO, cnv->options, soBytes));
4169
4170
    /* conversion loop */
4171
    /*
4172
     * This is another piece of ugly code:
4173
     * A goto into the loop if the converter state contains a first surrogate
4174
     * from the previous function call.
4175
     * It saves me to check in each loop iteration a check of if(c==0)
4176
     * and duplicating the trail-surrogate-handling code in the else
4177
     * branch of that check.
4178
     * I could not find any other way to get around this other than
4179
     * using a function call for the conversion and callback, which would
4180
     * be even more inefficient.
4181
     *
4182
     * Markus Scherer 2000-jul-19
4183
     */
4184
0
    if(c!=0 && targetCapacity>0) {
4185
0
        goto getTrail;
4186
0
    }
4187
4188
0
    while(source<sourceLimit) {
4189
        /*
4190
         * This following test is to see if available input would overflow the output.
4191
         * It does not catch output of more than one byte that
4192
         * overflows as a result of a multi-byte character or callback output
4193
         * from the last source character.
4194
         * Therefore, those situations also test for overflows and will
4195
         * then break the loop, too.
4196
         */
4197
0
        if(targetCapacity>0) {
4198
            /*
4199
             * Get a correct Unicode code point:
4200
             * a single UChar for a BMP code point or
4201
             * a matched surrogate pair for a "supplementary code point".
4202
             */
4203
0
            c=*source++;
4204
0
            ++nextSourceIndex;
4205
0
            if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) {
4206
0
                *target++=(uint8_t)c;
4207
0
                if(offsets!=NULL) {
4208
0
                    *offsets++=sourceIndex;
4209
0
                    prevSourceIndex=sourceIndex;
4210
0
                    sourceIndex=nextSourceIndex;
4211
0
                }
4212
0
                --targetCapacity;
4213
0
                c=0;
4214
0
                continue;
4215
0
            }
4216
            /*
4217
             * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX
4218
             * to avoid dealing with surrogates.
4219
             * MBCS_FAST_MAX must be >=0xd7ff.
4220
             */
4221
0
            if(c<=0xd7ff && mbcsIndex!=NULL) {
4222
0
                value=mbcsIndex[c>>6];
4223
4224
                /* get the bytes and the length for the output (copied from below and adapted for utf8Friendly data) */
4225
                /* There are only roundtrips (!=0) and no-mapping (==0) entries. */
4226
0
                switch(outputType) {
4227
0
                case MBCS_OUTPUT_2:
4228
0
                    value=((const uint16_t *)bytes)[value +(c&0x3f)];
4229
0
                    if(value<=0xff) {
4230
0
                        if(value==0) {
4231
0
                            goto unassigned;
4232
0
                        } else {
4233
0
                            length=1;
4234
0
                        }
4235
0
                    } else {
4236
0
                        length=2;
4237
0
                    }
4238
0
                    break;
4239
0
                case MBCS_OUTPUT_2_SISO:
4240
                    /* 1/2-byte stateful with Shift-In/Shift-Out */
4241
                    /*
4242
                     * Save the old state in the converter object
4243
                     * right here, then change the local prevLength state variable if necessary.
4244
                     * Then, if this character turns out to be unassigned or a fallback that
4245
                     * is not taken, the callback code must not save the new state in the converter
4246
                     * because the new state is for a character that is not output.
4247
                     * However, the callback must still restore the state from the converter
4248
                     * in case the callback function changed it for its output.
4249
                     */
4250
0
                    cnv->fromUnicodeStatus=prevLength; /* save the old state */
4251
0
                    value=((const uint16_t *)bytes)[value +(c&0x3f)];
4252
0
                    if(value<=0xff) {
4253
0
                        if(value==0) {
4254
0
                            goto unassigned;
4255
0
                        } else if(prevLength<=1) {
4256
0
                            length=1;
4257
0
                        } else {
4258
                            /* change from double-byte mode to single-byte */
4259
0
                            if (siLength == 1) {
4260
0
                                value|=(uint32_t)siBytes[0]<<8;
4261
0
                                length = 2;
4262
0
                            } else if (siLength == 2) {
4263
0
                                value|=(uint32_t)siBytes[1]<<8;
4264
0
                                value|=(uint32_t)siBytes[0]<<16;
4265
0
                                length = 3;
4266
0
                            }
4267
0
                            prevLength=1;
4268
0
                        }
4269
0
                    } else {
4270
0
                        if(prevLength==2) {
4271
0
                            length=2;
4272
0
                        } else {
4273
                            /* change from single-byte mode to double-byte */
4274
0
                            if (soLength == 1) {
4275
0
                                value|=(uint32_t)soBytes[0]<<16;
4276
0
                                length = 3;
4277
0
                            } else if (soLength == 2) {
4278
0
                                value|=(uint32_t)soBytes[1]<<16;
4279
0
                                value|=(uint32_t)soBytes[0]<<24;
4280
0
                                length = 4;
4281
0
                            }
4282
0
                            prevLength=2;
4283
0
                        }
4284
0
                    }
4285
0
                    break;
4286
0
                case MBCS_OUTPUT_DBCS_ONLY:
4287
                    /* table with single-byte results, but only DBCS mappings used */
4288
0
                    value=((const uint16_t *)bytes)[value +(c&0x3f)];
4289
0
                    if(value<=0xff) {
4290
                        /* no mapping or SBCS result, not taken for DBCS-only */
4291
0
                        goto unassigned;
4292
0
                    } else {
4293
0
                        length=2;
4294
0
                    }
4295
0
                    break;
4296
0
                case MBCS_OUTPUT_3:
4297
0
                    p=bytes+(value+(c&0x3f))*3;
4298
0
                    value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
4299
0
                    if(value<=0xff) {
4300
0
                        if(value==0) {
4301
0
                            goto unassigned;
4302
0
                        } else {
4303
0
                            length=1;
4304
0
                        }
4305
0
                    } else if(value<=0xffff) {
4306
0
                        length=2;
4307
0
                    } else {
4308
0
                        length=3;
4309
0
                    }
4310
0
                    break;
4311
0
                case MBCS_OUTPUT_4:
4312
0
                    value=((const uint32_t *)bytes)[value +(c&0x3f)];
4313
0
                    if(value<=0xff) {
4314
0
                        if(value==0) {
4315
0
                            goto unassigned;
4316
0
                        } else {
4317
0
                            length=1;
4318
0
                        }
4319
0
                    } else if(value<=0xffff) {
4320
0
                        length=2;
4321
0
                    } else if(value<=0xffffff) {
4322
0
                        length=3;
4323
0
                    } else {
4324
0
                        length=4;
4325
0
                    }
4326
0
                    break;
4327
0
                case MBCS_OUTPUT_3_EUC:
4328
0
                    value=((const uint16_t *)bytes)[value +(c&0x3f)];
4329
                    /* EUC 16-bit fixed-length representation */
4330
0
                    if(value<=0xff) {
4331
0
                        if(value==0) {
4332
0
                            goto unassigned;
4333
0
                        } else {
4334
0
                            length=1;
4335
0
                        }
4336
0
                    } else if((value&0x8000)==0) {
4337
0
                        value|=0x8e8000;
4338
0
                        length=3;
4339
0
                    } else if((value&0x80)==0) {
4340
0
                        value|=0x8f0080;
4341
0
                        length=3;
4342
0
                    } else {
4343
0
                        length=2;
4344
0
                    }
4345
0
                    break;
4346
0
                case MBCS_OUTPUT_4_EUC:
4347
0
                    p=bytes+(value+(c&0x3f))*3;
4348
0
                    value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
4349
                    /* EUC 16-bit fixed-length representation applied to the first two bytes */
4350
0
                    if(value<=0xff) {
4351
0
                        if(value==0) {
4352
0
                            goto unassigned;
4353
0
                        } else {
4354
0
                            length=1;
4355
0
                        }
4356
0
                    } else if(value<=0xffff) {
4357
0
                        length=2;
4358
0
                    } else if((value&0x800000)==0) {
4359
0
                        value|=0x8e800000;
4360
0
                        length=4;
4361
0
                    } else if((value&0x8000)==0) {
4362
0
                        value|=0x8f008000;
4363
0
                        length=4;
4364
0
                    } else {
4365
0
                        length=3;
4366
0
                    }
4367
0
                    break;
4368
0
                default:
4369
                    /* must not occur */
4370
                    /*
4371
                     * To avoid compiler warnings that value & length may be
4372
                     * used without having been initialized, we set them here.
4373
                     * In reality, this is unreachable code.
4374
                     * Not having a default branch also causes warnings with
4375
                     * some compilers.
4376
                     */
4377
0
                    value=0;
4378
0
                    length=0;
4379
0
                    break;
4380
0
                }
4381
                /* output the value */
4382
0
            } else {
4383
                /*
4384
                 * This also tests if the codepage maps single surrogates.
4385
                 * If it does, then surrogates are not paired but mapped separately.
4386
                 * Note that in this case unmatched surrogates are not detected.
4387
                 */
4388
0
                if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) {
4389
0
                    if(U16_IS_SURROGATE_LEAD(c)) {
4390
0
getTrail:
4391
0
                        if(source<sourceLimit) {
4392
                            /* test the following code unit */
4393
0
                            UChar trail=*source;
4394
0
                            if(U16_IS_TRAIL(trail)) {
4395
0
                                ++source;
4396
0
                                ++nextSourceIndex;
4397
0
                                c=U16_GET_SUPPLEMENTARY(c, trail);
4398
0
                                if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
4399
                                    /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
4400
0
                                    cnv->fromUnicodeStatus=prevLength; /* save the old state */
4401
                                    /* callback(unassigned) */
4402
0
                                    goto unassigned;
4403
0
                                }
4404
                                /* convert this supplementary code point */
4405
                                /* exit this condition tree */
4406
0
                            } else {
4407
                                /* this is an unmatched lead code unit (1st surrogate) */
4408
                                /* callback(illegal) */
4409
0
                                *pErrorCode=U_ILLEGAL_CHAR_FOUND;
4410
0
                                break;
4411
0
                            }
4412
0
                        } else {
4413
                            /* no more input */
4414
0
                            break;
4415
0
                        }
4416
0
                    } else {
4417
                        /* this is an unmatched trail code unit (2nd surrogate) */
4418
                        /* callback(illegal) */
4419
0
                        *pErrorCode=U_ILLEGAL_CHAR_FOUND;
4420
0
                        break;
4421
0
                    }
4422
0
                }
4423
4424
                /* convert the Unicode code point in c into codepage bytes */
4425
4426
                /*
4427
                 * The basic lookup is a triple-stage compact array (trie) lookup.
4428
                 * For details see the beginning of this file.
4429
                 *
4430
                 * Single-byte codepages are handled with a different data structure
4431
                 * by _MBCSSingle... functions.
4432
                 *
4433
                 * The result consists of a 32-bit value from stage 2 and
4434
                 * a pointer to as many bytes as are stored per character.
4435
                 * The pointer points to the character's bytes in stage 3.
4436
                 * Bits 15..0 of the stage 2 entry contain the stage 3 index
4437
                 * for that pointer, while bits 31..16 are flags for which of
4438
                 * the 16 characters in the block are roundtrip-assigned.
4439
                 *
4440
                 * For 2-byte and 4-byte codepages, the bytes are stored as uint16_t
4441
                 * respectively as uint32_t, in the platform encoding.
4442
                 * For 3-byte codepages, the bytes are always stored in big-endian order.
4443
                 *
4444
                 * For EUC encodings that use only either 0x8e or 0x8f as the first
4445
                 * byte of their longest byte sequences, the first two bytes in
4446
                 * this third stage indicate with their 7th bits whether these bytes
4447
                 * are to be written directly or actually need to be preceded by
4448
                 * one of the two Single-Shift codes. With this, the third stage
4449
                 * stores one byte fewer per character than the actual maximum length of
4450
                 * EUC byte sequences.
4451
                 *
4452
                 * Other than that, leading zero bytes are removed and the other
4453
                 * bytes output. A single zero byte may be output if the "assigned"
4454
                 * bit in stage 2 was on.
4455
                 * The data structure does not support zero byte output as a fallback,
4456
                 * and also does not allow output of leading zeros.
4457
                 */
4458
0
                stage2Entry=MBCS_STAGE_2_FROM_U(table, c);
4459
4460
                /* get the bytes and the length for the output */
4461
0
                switch(outputType) {
4462
0
                case MBCS_OUTPUT_2:
4463
0
                    value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c);
4464
0
                    if(value<=0xff) {
4465
0
                        length=1;
4466
0
                    } else {
4467
0
                        length=2;
4468
0
                    }
4469
0
                    break;
4470
0
                case MBCS_OUTPUT_2_SISO:
4471
                    /* 1/2-byte stateful with Shift-In/Shift-Out */
4472
                    /*
4473
                     * Save the old state in the converter object
4474
                     * right here, then change the local prevLength state variable if necessary.
4475
                     * Then, if this character turns out to be unassigned or a fallback that
4476
                     * is not taken, the callback code must not save the new state in the converter
4477
                     * because the new state is for a character that is not output.
4478
                     * However, the callback must still restore the state from the converter
4479
                     * in case the callback function changed it for its output.
4480
                     */
4481
0
                    cnv->fromUnicodeStatus=prevLength; /* save the old state */
4482
0
                    value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c);
4483
0
                    if(value<=0xff) {
4484
0
                        if(value==0 && MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)==0) {
4485
                            /* no mapping, leave value==0 */
4486
0
                            length=0;
4487
0
                        } else if(prevLength<=1) {
4488
0
                            length=1;
4489
0
                        } else {
4490
                            /* change from double-byte mode to single-byte */
4491
0
                            if (siLength == 1) {
4492
0
                                value|=(uint32_t)siBytes[0]<<8;
4493
0
                                length = 2;
4494
0
                            } else if (siLength == 2) {
4495
0
                                value|=(uint32_t)siBytes[1]<<8;
4496
0
                                value|=(uint32_t)siBytes[0]<<16;
4497
0
                                length = 3;
4498
0
                            }
4499
0
                            prevLength=1;
4500
0
                        }
4501
0
                    } else {
4502
0
                        if(prevLength==2) {
4503
0
                            length=2;
4504
0
                        } else {
4505
                            /* change from single-byte mode to double-byte */
4506
0
                            if (soLength == 1) {
4507
0
                                value|=(uint32_t)soBytes[0]<<16;
4508
0
                                length = 3;
4509
0
                            } else if (soLength == 2) {
4510
0
                                value|=(uint32_t)soBytes[1]<<16;
4511
0
                                value|=(uint32_t)soBytes[0]<<24;
4512
0
                                length = 4;
4513
0
                            }
4514
0
                            prevLength=2;
4515
0
                        }
4516
0
                    }
4517
0
                    break;
4518
0
                case MBCS_OUTPUT_DBCS_ONLY:
4519
                    /* table with single-byte results, but only DBCS mappings used */
4520
0
                    value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c);
4521
0
                    if(value<=0xff) {
4522
                        /* no mapping or SBCS result, not taken for DBCS-only */
4523
0
                        value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */
4524
0
                        length=0;
4525
0
                    } else {
4526
0
                        length=2;
4527
0
                    }
4528
0
                    break;
4529
0
                case MBCS_OUTPUT_3:
4530
0
                    p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c);
4531
0
                    value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
4532
0
                    if(value<=0xff) {
4533
0
                        length=1;
4534
0
                    } else if(value<=0xffff) {
4535
0
                        length=2;
4536
0
                    } else {
4537
0
                        length=3;
4538
0
                    }
4539
0
                    break;
4540
0
                case MBCS_OUTPUT_4:
4541
0
                    value=MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c);
4542
0
                    if(value<=0xff) {
4543
0
                        length=1;
4544
0
                    } else if(value<=0xffff) {
4545
0
                        length=2;
4546
0
                    } else if(value<=0xffffff) {
4547
0
                        length=3;
4548
0
                    } else {
4549
0
                        length=4;
4550
0
                    }
4551
0
                    break;
4552
0
                case MBCS_OUTPUT_3_EUC:
4553
0
                    value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c);
4554
                    /* EUC 16-bit fixed-length representation */
4555
0
                    if(value<=0xff) {
4556
0
                        length=1;
4557
0
                    } else if((value&0x8000)==0) {
4558
0
                        value|=0x8e8000;
4559
0
                        length=3;
4560
0
                    } else if((value&0x80)==0) {
4561
0
                        value|=0x8f0080;
4562
0
                        length=3;
4563
0
                    } else {
4564
0
                        length=2;
4565
0
                    }
4566
0
                    break;
4567
0
                case MBCS_OUTPUT_4_EUC:
4568
0
                    p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c);
4569
0
                    value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
4570
                    /* EUC 16-bit fixed-length representation applied to the first two bytes */
4571
0
                    if(value<=0xff) {
4572
0
                        length=1;
4573
0
                    } else if(value<=0xffff) {
4574
0
                        length=2;
4575
0
                    } else if((value&0x800000)==0) {
4576
0
                        value|=0x8e800000;
4577
0
                        length=4;
4578
0
                    } else if((value&0x8000)==0) {
4579
0
                        value|=0x8f008000;
4580
0
                        length=4;
4581
0
                    } else {
4582
0
                        length=3;
4583
0
                    }
4584
0
                    break;
4585
0
                default:
4586
                    /* must not occur */
4587
                    /*
4588
                     * To avoid compiler warnings that value & length may be
4589
                     * used without having been initialized, we set them here.
4590
                     * In reality, this is unreachable code.
4591
                     * Not having a default branch also causes warnings with
4592
                     * some compilers.
4593
                     */
4594
0
                    value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */
4595
0
                    length=0;
4596
0
                    break;
4597
0
                }
4598
4599
                /* is this code point assigned, or do we use fallbacks? */
4600
0
                if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)!=0 ||
4601
0
                     (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0))
4602
0
                ) {
4603
                    /*
4604
                     * We allow a 0 byte output if the "assigned" bit is set for this entry.
4605
                     * There is no way with this data structure for fallback output
4606
                     * to be a zero byte.
4607
                     */
4608
4609
0
unassigned:
4610
                    /* try an extension mapping */
4611
0
                    pArgs->source=source;
4612
0
                    c=_extFromU(cnv, cnv->sharedData,
4613
0
                                c, &source, sourceLimit,
4614
0
                                &target, target+targetCapacity,
4615
0
                                &offsets, sourceIndex,
4616
0
                                pArgs->flush,
4617
0
                                pErrorCode);
4618
0
                    nextSourceIndex+=(int32_t)(source-pArgs->source);
4619
0
                    prevLength=cnv->fromUnicodeStatus; /* restore SISO state */
4620
4621
0
                    if(U_FAILURE(*pErrorCode)) {
4622
                        /* not mappable or buffer overflow */
4623
0
                        break;
4624
0
                    } else {
4625
                        /* a mapping was written to the target, continue */
4626
4627
                        /* recalculate the targetCapacity after an extension mapping */
4628
0
                        targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target);
4629
4630
                        /* normal end of conversion: prepare for a new character */
4631
0
                        if(offsets!=NULL) {
4632
0
                            prevSourceIndex=sourceIndex;
4633
0
                            sourceIndex=nextSourceIndex;
4634
0
                        }
4635
0
                        continue;
4636
0
                    }
4637
0
                }
4638
0
            }
4639
4640
            /* write the output character bytes from value and length */
4641
            /* from the first if in the loop we know that targetCapacity>0 */
4642
0
            if(length<=targetCapacity) {
4643
0
                if(offsets==NULL) {
4644
0
                    switch(length) {
4645
                        /* each branch falls through to the next one */
4646
0
                    case 4:
4647
0
                        *target++=(uint8_t)(value>>24);
4648
0
                        U_FALLTHROUGH;
4649
0
                    case 3:
4650
0
                        *target++=(uint8_t)(value>>16);
4651
0
                        U_FALLTHROUGH;
4652
0
                    case 2:
4653
0
                        *target++=(uint8_t)(value>>8);
4654
0
                        U_FALLTHROUGH;
4655
0
                    case 1:
4656
0
                        *target++=(uint8_t)value;
4657
0
                        U_FALLTHROUGH;
4658
0
                    default:
4659
                        /* will never occur */
4660
0
                        break;
4661
0
                    }
4662
0
                } else {
4663
0
                    switch(length) {
4664
                        /* each branch falls through to the next one */
4665
0
                    case 4:
4666
0
                        *target++=(uint8_t)(value>>24);
4667
0
                        *offsets++=sourceIndex;
4668
0
                        U_FALLTHROUGH;
4669
0
                    case 3:
4670
0
                        *target++=(uint8_t)(value>>16);
4671
0
                        *offsets++=sourceIndex;
4672
0
                        U_FALLTHROUGH;
4673
0
                    case 2:
4674
0
                        *target++=(uint8_t)(value>>8);
4675
0
                        *offsets++=sourceIndex;
4676
0
                        U_FALLTHROUGH;
4677
0
                    case 1:
4678
0
                        *target++=(uint8_t)value;
4679
0
                        *offsets++=sourceIndex;
4680
0
                        U_FALLTHROUGH;
4681
0
                    default:
4682
                        /* will never occur */
4683
0
                        break;
4684
0
                    }
4685
0
                }
4686
0
                targetCapacity-=length;
4687
0
            } else {
4688
0
                uint8_t *charErrorBuffer;
4689
4690
                /*
4691
                 * We actually do this backwards here:
4692
                 * In order to save an intermediate variable, we output
4693
                 * first to the overflow buffer what does not fit into the
4694
                 * regular target.
4695
                 */
4696
                /* we know that 1<=targetCapacity<length<=4 */
4697
0
                length-=targetCapacity;
4698
0
                charErrorBuffer=(uint8_t *)cnv->charErrorBuffer;
4699
0
                switch(length) {
4700
                    /* each branch falls through to the next one */
4701
0
                case 3:
4702
0
                    *charErrorBuffer++=(uint8_t)(value>>16);
4703
0
                    U_FALLTHROUGH;
4704
0
                case 2:
4705
0
                    *charErrorBuffer++=(uint8_t)(value>>8);
4706
0
                    U_FALLTHROUGH;
4707
0
                case 1:
4708
0
                    *charErrorBuffer=(uint8_t)value;
4709
0
                    U_FALLTHROUGH;
4710
0
                default:
4711
                    /* will never occur */
4712
0
                    break;
4713
0
                }
4714
0
                cnv->charErrorBufferLength=(int8_t)length;
4715
4716
                /* now output what fits into the regular target */
4717
0
                value>>=8*length; /* length was reduced by targetCapacity */
4718
0
                switch(targetCapacity) {
4719
                    /* each branch falls through to the next one */
4720
0
                case 3:
4721
0
                    *target++=(uint8_t)(value>>16);
4722
0
                    if(offsets!=NULL) {
4723
0
                        *offsets++=sourceIndex;
4724
0
                    }
4725
0
                    U_FALLTHROUGH;
4726
0
                case 2:
4727
0
                    *target++=(uint8_t)(value>>8);
4728
0
                    if(offsets!=NULL) {
4729
0
                        *offsets++=sourceIndex;
4730
0
                    }
4731
0
                    U_FALLTHROUGH;
4732
0
                case 1:
4733
0
                    *target++=(uint8_t)value;
4734
0
                    if(offsets!=NULL) {
4735
0
                        *offsets++=sourceIndex;
4736
0
                    }
4737
0
                    U_FALLTHROUGH;
4738
0
                default:
4739
                    /* will never occur */
4740
0
                    break;
4741
0
                }
4742
4743
                /* target overflow */
4744
0
                targetCapacity=0;
4745
0
                *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
4746
0
                c=0;
4747
0
                break;
4748
0
            }
4749
4750
            /* normal end of conversion: prepare for a new character */
4751
0
            c=0;
4752
0
            if(offsets!=NULL) {
4753
0
                prevSourceIndex=sourceIndex;
4754
0
                sourceIndex=nextSourceIndex;
4755
0
            }
4756
0
            continue;
4757
0
        } else {
4758
            /* target is full */
4759
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
4760
0
            break;
4761
0
        }
4762
0
    }
4763
4764
    /*
4765
     * the end of the input stream and detection of truncated input
4766
     * are handled by the framework, but for EBCDIC_STATEFUL conversion
4767
     * we need to emit an SI at the very end
4768
     *
4769
     * conditions:
4770
     *   successful
4771
     *   EBCDIC_STATEFUL in DBCS mode
4772
     *   end of input and no truncated input
4773
     */
4774
0
    if( U_SUCCESS(*pErrorCode) &&
4775
0
        outputType==MBCS_OUTPUT_2_SISO && prevLength==2 &&
4776
0
        pArgs->flush && source>=sourceLimit && c==0
4777
0
    ) {
4778
        /* EBCDIC_STATEFUL ending with DBCS: emit an SI to return the output stream to SBCS */
4779
0
        if(targetCapacity>0) {
4780
0
            *target++=(uint8_t)siBytes[0];
4781
0
            if (siLength == 2) {
4782
0
                if (targetCapacity<2) {
4783
0
                    cnv->charErrorBuffer[0]=(uint8_t)siBytes[1];
4784
0
                    cnv->charErrorBufferLength=1;
4785
0
                    *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
4786
0
                } else {
4787
0
                    *target++=(uint8_t)siBytes[1];
4788
0
                }
4789
0
            }
4790
0
            if(offsets!=NULL) {
4791
                /* set the last source character's index (sourceIndex points at sourceLimit now) */
4792
0
                *offsets++=prevSourceIndex;
4793
0
            }
4794
0
        } else {
4795
            /* target is full */
4796
0
            cnv->charErrorBuffer[0]=(uint8_t)siBytes[0];
4797
0
            if (siLength == 2) {
4798
0
                cnv->charErrorBuffer[1]=(uint8_t)siBytes[1];
4799
0
            }
4800
0
            cnv->charErrorBufferLength=siLength;
4801
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
4802
0
        }
4803
0
        prevLength=1; /* we switched into SBCS */
4804
0
    }
4805
4806
    /* set the converter state back into UConverter */
4807
0
    cnv->fromUChar32=c;
4808
0
    cnv->fromUnicodeStatus=prevLength;
4809
4810
    /* write back the updated pointers */
4811
0
    pArgs->source=source;
4812
0
    pArgs->target=(char *)target;
4813
0
    pArgs->offsets=offsets;
4814
0
}
4815
4816
/*
4817
 * This is another simple conversion function for internal use by other
4818
 * conversion implementations.
4819
 * It does not use the converter state nor call callbacks.
4820
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
4821
 * It handles conversion extensions but not GB 18030.
4822
 *
4823
 * It converts one single Unicode code point into codepage bytes, encoded
4824
 * as one 32-bit value. The function returns the number of bytes in *pValue:
4825
 * 1..4 the number of bytes in *pValue
4826
 * 0    unassigned (*pValue undefined)
4827
 * -1   illegal (currently not used, *pValue undefined)
4828
 *
4829
 * *pValue will contain the resulting bytes with the last byte in bits 7..0,
4830
 * the second to last byte in bits 15..8, etc.
4831
 * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
4832
 */
4833
U_CFUNC int32_t
4834
ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
4835
                 UChar32 c, uint32_t *pValue,
4836
0
                 UBool useFallback) {
4837
0
    const int32_t *cx;
4838
0
    const uint16_t *table;
4839
#if 0
4840
/* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */
4841
    const uint8_t *p;
4842
#endif
4843
0
    uint32_t stage2Entry;
4844
0
    uint32_t value;
4845
0
    int32_t length;
4846
4847
    /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
4848
0
    if(c<=0xffff || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
4849
0
        table=sharedData->mbcs.fromUnicodeTable;
4850
4851
        /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */
4852
0
        if(sharedData->mbcs.outputType==MBCS_OUTPUT_1) {
4853
0
            value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c);
4854
            /* is this code point assigned, or do we use fallbacks? */
4855
0
            if(useFallback ? value>=0x800 : value>=0xc00) {
4856
0
                *pValue=value&0xff;
4857
0
                return 1;
4858
0
            }
4859
0
        } else /* outputType!=MBCS_OUTPUT_1 */ {
4860
0
            stage2Entry=MBCS_STAGE_2_FROM_U(table, c);
4861
4862
            /* get the bytes and the length for the output */
4863
0
            switch(sharedData->mbcs.outputType) {
4864
0
            case MBCS_OUTPUT_2:
4865
0
                value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
4866
0
                if(value<=0xff) {
4867
0
                    length=1;
4868
0
                } else {
4869
0
                    length=2;
4870
0
                }
4871
0
                break;
4872
#if 0
4873
/* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */
4874
            case MBCS_OUTPUT_DBCS_ONLY:
4875
                /* table with single-byte results, but only DBCS mappings used */
4876
                value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
4877
                if(value<=0xff) {
4878
                    /* no mapping or SBCS result, not taken for DBCS-only */
4879
                    value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */
4880
                    length=0;
4881
                } else {
4882
                    length=2;
4883
                }
4884
                break;
4885
            case MBCS_OUTPUT_3:
4886
                p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
4887
                value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
4888
                if(value<=0xff) {
4889
                    length=1;
4890
                } else if(value<=0xffff) {
4891
                    length=2;
4892
                } else {
4893
                    length=3;
4894
                }
4895
                break;
4896
            case MBCS_OUTPUT_4:
4897
                value=MBCS_VALUE_4_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
4898
                if(value<=0xff) {
4899
                    length=1;
4900
                } else if(value<=0xffff) {
4901
                    length=2;
4902
                } else if(value<=0xffffff) {
4903
                    length=3;
4904
                } else {
4905
                    length=4;
4906
                }
4907
                break;
4908
            case MBCS_OUTPUT_3_EUC:
4909
                value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
4910
                /* EUC 16-bit fixed-length representation */
4911
                if(value<=0xff) {
4912
                    length=1;
4913
                } else if((value&0x8000)==0) {
4914
                    value|=0x8e8000;
4915
                    length=3;
4916
                } else if((value&0x80)==0) {
4917
                    value|=0x8f0080;
4918
                    length=3;
4919
                } else {
4920
                    length=2;
4921
                }
4922
                break;
4923
            case MBCS_OUTPUT_4_EUC:
4924
                p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
4925
                value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
4926
                /* EUC 16-bit fixed-length representation applied to the first two bytes */
4927
                if(value<=0xff) {
4928
                    length=1;
4929
                } else if(value<=0xffff) {
4930
                    length=2;
4931
                } else if((value&0x800000)==0) {
4932
                    value|=0x8e800000;
4933
                    length=4;
4934
                } else if((value&0x8000)==0) {
4935
                    value|=0x8f008000;
4936
                    length=4;
4937
                } else {
4938
                    length=3;
4939
                }
4940
                break;
4941
#endif
4942
0
            default:
4943
                /* must not occur */
4944
0
                return -1;
4945
0
            }
4946
4947
            /* is this code point assigned, or do we use fallbacks? */
4948
0
            if( MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ||
4949
0
                (FROM_U_USE_FALLBACK(useFallback, c) && value!=0)
4950
0
            ) {
4951
                /*
4952
                 * We allow a 0 byte output if the "assigned" bit is set for this entry.
4953
                 * There is no way with this data structure for fallback output
4954
                 * to be a zero byte.
4955
                 */
4956
                /* assigned */
4957
0
                *pValue=value;
4958
0
                return length;
4959
0
            }
4960
0
        }
4961
0
    }
4962
4963
0
    cx=sharedData->mbcs.extIndexes;
4964
0
    if(cx!=NULL) {
4965
0
        length=ucnv_extSimpleMatchFromU(cx, c, pValue, useFallback);
4966
0
        return length>=0 ? length : -length;  /* return abs(length); */
4967
0
    }
4968
4969
    /* unassigned */
4970
0
    return 0;
4971
0
}
4972
4973
4974
#if 0
4975
/*
4976
 * This function has been moved to ucnv2022.c for inlining.
4977
 * This implementation is here only for documentation purposes
4978
 */
4979
4980
/**
4981
 * This version of ucnv_MBCSFromUChar32() is optimized for single-byte codepages.
4982
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
4983
 * It does not handle conversion extensions (_extFromU()).
4984
 *
4985
 * It returns the codepage byte for the code point, or -1 if it is unassigned.
4986
 */
4987
U_CFUNC int32_t
4988
ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
4989
                       UChar32 c,
4990
                       UBool useFallback) {
4991
    const uint16_t *table;
4992
    int32_t value;
4993
4994
    /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
4995
    if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
4996
        return -1;
4997
    }
4998
4999
    /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */
5000
    table=sharedData->mbcs.fromUnicodeTable;
5001
5002
    /* get the byte for the output */
5003
    value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c);
5004
    /* is this code point assigned, or do we use fallbacks? */
5005
    if(useFallback ? value>=0x800 : value>=0xc00) {
5006
        return value&0xff;
5007
    } else {
5008
        return -1;
5009
    }
5010
}
5011
#endif
5012
5013
/* MBCS-from-UTF-8 conversion functions ------------------------------------- */
5014
5015
/* offsets for n-byte UTF-8 sequences that were calculated with ((lead<<6)+trail)<<6+trail... */
5016
static const UChar32
5017
utf8_offsets[5]={ 0, 0, 0x3080, 0xE2080, 0x3C82080 };
5018
5019
static void U_CALLCONV
5020
ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
5021
                  UConverterToUnicodeArgs *pToUArgs,
5022
0
                  UErrorCode *pErrorCode) {
5023
0
    UConverter *utf8, *cnv;
5024
0
    const uint8_t *source, *sourceLimit;
5025
0
    uint8_t *target;
5026
0
    int32_t targetCapacity;
5027
5028
0
    const uint16_t *table, *sbcsIndex;
5029
0
    const uint16_t *results;
5030
5031
0
    int8_t oldToULength, toULength, toULimit;
5032
5033
0
    UChar32 c;
5034
0
    uint8_t b, t1, t2;
5035
5036
0
    uint32_t asciiRoundtrips;
5037
0
    uint16_t value, minValue = 0;
5038
0
    UBool hasSupplementary;
5039
5040
    /* set up the local pointers */
5041
0
    utf8=pToUArgs->converter;
5042
0
    cnv=pFromUArgs->converter;
5043
0
    source=(uint8_t *)pToUArgs->source;
5044
0
    sourceLimit=(uint8_t *)pToUArgs->sourceLimit;
5045
0
    target=(uint8_t *)pFromUArgs->target;
5046
0
    targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target);
5047
5048
0
    table=cnv->sharedData->mbcs.fromUnicodeTable;
5049
0
    sbcsIndex=cnv->sharedData->mbcs.sbcsIndex;
5050
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
5051
0
        results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes;
5052
0
    } else {
5053
0
        results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes;
5054
0
    }
5055
0
    asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips;
5056
5057
0
    if(cnv->useFallback) {
5058
        /* use all roundtrip and fallback results */
5059
0
        minValue=0x800;
5060
0
    } else {
5061
        /* use only roundtrips and fallbacks from private-use characters */
5062
0
        minValue=0xc00;
5063
0
    }
5064
0
    hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY);
5065
5066
    /* get the converter state from the UTF-8 UConverter */
5067
0
    if(utf8->toULength > 0) {
5068
0
        toULength=oldToULength=utf8->toULength;
5069
0
        toULimit=(int8_t)utf8->mode;
5070
0
        c=(UChar32)utf8->toUnicodeStatus;
5071
0
    } else {
5072
0
        toULength=oldToULength=toULimit=0;
5073
0
        c = 0;
5074
0
    }
5075
5076
    // The conversion loop checks source<sourceLimit only once per 1/2/3-byte character.
5077
    // If the buffer ends with a truncated 2- or 3-byte sequence,
5078
    // then we reduce the sourceLimit to before that,
5079
    // and collect the remaining bytes after the conversion loop.
5080
0
    {
5081
        // Do not go back into the bytes that will be read for finishing a partial
5082
        // sequence from the previous buffer.
5083
0
        int32_t length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength);
5084
0
        if(length>0) {
5085
0
            uint8_t b1=*(sourceLimit-1);
5086
0
            if(U8_IS_SINGLE(b1)) {
5087
                // common ASCII character
5088
0
            } else if(U8_IS_TRAIL(b1) && length>=2) {
5089
0
                uint8_t b2=*(sourceLimit-2);
5090
0
                if(0xe0<=b2 && b2<0xf0 && U8_IS_VALID_LEAD3_AND_T1(b2, b1)) {
5091
                    // truncated 3-byte sequence
5092
0
                    sourceLimit-=2;
5093
0
                }
5094
0
            } else if(0xc2<=b1 && b1<0xf0) {
5095
                // truncated 2- or 3-byte sequence
5096
0
                --sourceLimit;
5097
0
            }
5098
0
        }
5099
0
    }
5100
5101
0
    if(c!=0 && targetCapacity>0) {
5102
0
        utf8->toUnicodeStatus=0;
5103
0
        utf8->toULength=0;
5104
0
        goto moreBytes;
5105
        /*
5106
         * Note: We could avoid the goto by duplicating some of the moreBytes
5107
         * code, but only up to the point of collecting a complete UTF-8
5108
         * sequence; then recurse for the toUBytes[toULength]
5109
         * and then continue with normal conversion.
5110
         *
5111
         * If so, move this code to just after initializing the minimum
5112
         * set of local variables for reading the UTF-8 input
5113
         * (utf8, source, target, limits but not cnv, table, minValue, etc.).
5114
         *
5115
         * Potential advantages:
5116
         * - avoid the goto
5117
         * - oldToULength could become a local variable in just those code blocks
5118
         *   that deal with buffer boundaries
5119
         * - possibly faster if the goto prevents some compiler optimizations
5120
         *   (this would need measuring to confirm)
5121
         * Disadvantage:
5122
         * - code duplication
5123
         */
5124
0
    }
5125
5126
    /* conversion loop */
5127
0
    while(source<sourceLimit) {
5128
0
        if(targetCapacity>0) {
5129
0
            b=*source++;
5130
0
            if(U8_IS_SINGLE(b)) {
5131
                /* convert ASCII */
5132
0
                if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) {
5133
0
                    *target++=(uint8_t)b;
5134
0
                    --targetCapacity;
5135
0
                    continue;
5136
0
                } else {
5137
0
                    c=b;
5138
0
                    value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, 0, c);
5139
0
                }
5140
0
            } else {
5141
0
                if(b<0xe0) {
5142
0
                    if( /* handle U+0080..U+07FF inline */
5143
0
                        b>=0xc2 &&
5144
0
                        (t1=(uint8_t)(*source-0x80)) <= 0x3f
5145
0
                    ) {
5146
0
                        c=b&0x1f;
5147
0
                        ++source;
5148
0
                        value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t1);
5149
0
                        if(value>=minValue) {
5150
0
                            *target++=(uint8_t)value;
5151
0
                            --targetCapacity;
5152
0
                            continue;
5153
0
                        } else {
5154
0
                            c=(c<<6)|t1;
5155
0
                        }
5156
0
                    } else {
5157
0
                        c=-1;
5158
0
                    }
5159
0
                } else if(b==0xe0) {
5160
0
                    if( /* handle U+0800..U+0FFF inline */
5161
0
                        (t1=(uint8_t)(source[0]-0x80)) <= 0x3f && t1 >= 0x20 &&
5162
0
                        (t2=(uint8_t)(source[1]-0x80)) <= 0x3f
5163
0
                    ) {
5164
0
                        c=t1;
5165
0
                        source+=2;
5166
0
                        value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t2);
5167
0
                        if(value>=minValue) {
5168
0
                            *target++=(uint8_t)value;
5169
0
                            --targetCapacity;
5170
0
                            continue;
5171
0
                        } else {
5172
0
                            c=(c<<6)|t2;
5173
0
                        }
5174
0
                    } else {
5175
0
                        c=-1;
5176
0
                    }
5177
0
                } else {
5178
0
                    c=-1;
5179
0
                }
5180
5181
0
                if(c<0) {
5182
                    /* handle "complicated" and error cases, and continuing partial characters */
5183
0
                    oldToULength=0;
5184
0
                    toULength=1;
5185
0
                    toULimit=U8_COUNT_BYTES_NON_ASCII(b);
5186
0
                    c=b;
5187
0
moreBytes:
5188
0
                    while(toULength<toULimit) {
5189
                        /*
5190
                         * The sourceLimit may have been adjusted before the conversion loop
5191
                         * to stop before a truncated sequence.
5192
                         * Here we need to use the real limit in case we have two truncated
5193
                         * sequences at the end.
5194
                         * See ticket #7492.
5195
                         */
5196
0
                        if(source<(uint8_t *)pToUArgs->sourceLimit) {
5197
0
                            b=*source;
5198
0
                            if(icu::UTF8::isValidTrail(c, b, toULength, toULimit)) {
5199
0
                                ++source;
5200
0
                                ++toULength;
5201
0
                                c=(c<<6)+b;
5202
0
                            } else {
5203
0
                                break; /* sequence too short, stop with toULength<toULimit */
5204
0
                            }
5205
0
                        } else {
5206
                            /* store the partial UTF-8 character, compatible with the regular UTF-8 converter */
5207
0
                            source-=(toULength-oldToULength);
5208
0
                            while(oldToULength<toULength) {
5209
0
                                utf8->toUBytes[oldToULength++]=*source++;
5210
0
                            }
5211
0
                            utf8->toUnicodeStatus=c;
5212
0
                            utf8->toULength=toULength;
5213
0
                            utf8->mode=toULimit;
5214
0
                            pToUArgs->source=(char *)source;
5215
0
                            pFromUArgs->target=(char *)target;
5216
0
                            return;
5217
0
                        }
5218
0
                    }
5219
5220
0
                    if(toULength==toULimit) {
5221
0
                        c-=utf8_offsets[toULength];
5222
0
                        if(toULength<=3) {  /* BMP */
5223
0
                            value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
5224
0
                        } else {
5225
                            /* supplementary code point */
5226
0
                            if(!hasSupplementary) {
5227
                                /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
5228
0
                                value=0;
5229
0
                            } else {
5230
0
                                value=MBCS_SINGLE_RESULT_FROM_U(table, results, c);
5231
0
                            }
5232
0
                        }
5233
0
                    } else {
5234
                        /* error handling: illegal UTF-8 byte sequence */
5235
0
                        source-=(toULength-oldToULength);
5236
0
                        while(oldToULength<toULength) {
5237
0
                            utf8->toUBytes[oldToULength++]=*source++;
5238
0
                        }
5239
0
                        utf8->toULength=toULength;
5240
0
                        pToUArgs->source=(char *)source;
5241
0
                        pFromUArgs->target=(char *)target;
5242
0
                        *pErrorCode=U_ILLEGAL_CHAR_FOUND;
5243
0
                        return;
5244
0
                    }
5245
0
                }
5246
0
            }
5247
5248
0
            if(value>=minValue) {
5249
                /* output the mapping for c */
5250
0
                *target++=(uint8_t)value;
5251
0
                --targetCapacity;
5252
0
            } else {
5253
                /* value<minValue means c is unassigned (unmappable) */
5254
                /*
5255
                 * Try an extension mapping.
5256
                 * Pass in no source because we don't have UTF-16 input.
5257
                 * If we have a partial match on c, we will return and revert
5258
                 * to UTF-8->UTF-16->charset conversion.
5259
                 */
5260
0
                static const UChar nul=0;
5261
0
                const UChar *noSource=&nul;
5262
0
                c=_extFromU(cnv, cnv->sharedData,
5263
0
                            c, &noSource, noSource,
5264
0
                            &target, target+targetCapacity,
5265
0
                            NULL, -1,
5266
0
                            pFromUArgs->flush,
5267
0
                            pErrorCode);
5268
5269
0
                if(U_FAILURE(*pErrorCode)) {
5270
                    /* not mappable or buffer overflow */
5271
0
                    cnv->fromUChar32=c;
5272
0
                    break;
5273
0
                } else if(cnv->preFromUFirstCP>=0) {
5274
                    /*
5275
                     * Partial match, return and revert to pivoting.
5276
                     * In normal from-UTF-16 conversion, we would just continue
5277
                     * but then exit the loop because the extension match would
5278
                     * have consumed the source.
5279
                     */
5280
0
                    *pErrorCode=U_USING_DEFAULT_WARNING;
5281
0
                    break;
5282
0
                } else {
5283
                    /* a mapping was written to the target, continue */
5284
5285
                    /* recalculate the targetCapacity after an extension mapping */
5286
0
                    targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target);
5287
0
                }
5288
0
            }
5289
0
        } else {
5290
            /* target is full */
5291
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
5292
0
            break;
5293
0
        }
5294
0
    }
5295
5296
    /*
5297
     * The sourceLimit may have been adjusted before the conversion loop
5298
     * to stop before a truncated sequence.
5299
     * If so, then collect the truncated sequence now.
5300
     */
5301
0
    if(U_SUCCESS(*pErrorCode) &&
5302
0
            cnv->preFromUFirstCP<0 &&
5303
0
            source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) {
5304
0
        c=utf8->toUBytes[0]=b=*source++;
5305
0
        toULength=1;
5306
0
        toULimit=U8_COUNT_BYTES(b);
5307
0
        while(source<sourceLimit) {
5308
0
            utf8->toUBytes[toULength++]=b=*source++;
5309
0
            c=(c<<6)+b;
5310
0
        }
5311
0
        utf8->toUnicodeStatus=c;
5312
0
        utf8->toULength=toULength;
5313
0
        utf8->mode=toULimit;
5314
0
    }
5315
5316
    /* write back the updated pointers */
5317
0
    pToUArgs->source=(char *)source;
5318
0
    pFromUArgs->target=(char *)target;
5319
0
}
5320
5321
static void U_CALLCONV
5322
ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
5323
                  UConverterToUnicodeArgs *pToUArgs,
5324
0
                  UErrorCode *pErrorCode) {
5325
0
    UConverter *utf8, *cnv;
5326
0
    const uint8_t *source, *sourceLimit;
5327
0
    uint8_t *target;
5328
0
    int32_t targetCapacity;
5329
5330
0
    const uint16_t *table, *mbcsIndex;
5331
0
    const uint16_t *results;
5332
5333
0
    int8_t oldToULength, toULength, toULimit;
5334
5335
0
    UChar32 c;
5336
0
    uint8_t b, t1, t2;
5337
5338
0
    uint32_t stage2Entry;
5339
0
    uint32_t asciiRoundtrips;
5340
0
    uint16_t value = 0;
5341
0
    UBool hasSupplementary;
5342
5343
    /* set up the local pointers */
5344
0
    utf8=pToUArgs->converter;
5345
0
    cnv=pFromUArgs->converter;
5346
0
    source=(uint8_t *)pToUArgs->source;
5347
0
    sourceLimit=(uint8_t *)pToUArgs->sourceLimit;
5348
0
    target=(uint8_t *)pFromUArgs->target;
5349
0
    targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target);
5350
5351
0
    table=cnv->sharedData->mbcs.fromUnicodeTable;
5352
0
    mbcsIndex=cnv->sharedData->mbcs.mbcsIndex;
5353
0
    if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
5354
0
        results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes;
5355
0
    } else {
5356
0
        results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes;
5357
0
    }
5358
0
    asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips;
5359
5360
0
    hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY);
5361
5362
    /* get the converter state from the UTF-8 UConverter */
5363
0
    if(utf8->toULength > 0) {
5364
0
        toULength=oldToULength=utf8->toULength;
5365
0
        toULimit=(int8_t)utf8->mode;
5366
0
        c=(UChar32)utf8->toUnicodeStatus;
5367
0
    } else {
5368
0
        toULength=oldToULength=toULimit=0;
5369
0
        c = 0;
5370
0
    }
5371
5372
    // The conversion loop checks source<sourceLimit only once per 1/2/3-byte character.
5373
    // If the buffer ends with a truncated 2- or 3-byte sequence,
5374
    // then we reduce the sourceLimit to before that,
5375
    // and collect the remaining bytes after the conversion loop.
5376
0
    {
5377
        // Do not go back into the bytes that will be read for finishing a partial
5378
        // sequence from the previous buffer.
5379
0
        int32_t length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength);
5380
0
        if(length>0) {
5381
0
            uint8_t b1=*(sourceLimit-1);
5382
0
            if(U8_IS_SINGLE(b1)) {
5383
                // common ASCII character
5384
0
            } else if(U8_IS_TRAIL(b1) && length>=2) {
5385
0
                uint8_t b2=*(sourceLimit-2);
5386
0
                if(0xe0<=b2 && b2<0xf0 && U8_IS_VALID_LEAD3_AND_T1(b2, b1)) {
5387
                    // truncated 3-byte sequence
5388
0
                    sourceLimit-=2;
5389
0
                }
5390
0
            } else if(0xc2<=b1 && b1<0xf0) {
5391
                // truncated 2- or 3-byte sequence
5392
0
                --sourceLimit;
5393
0
            }
5394
0
        }
5395
0
    }
5396
5397
0
    if(c!=0 && targetCapacity>0) {
5398
0
        utf8->toUnicodeStatus=0;
5399
0
        utf8->toULength=0;
5400
0
        goto moreBytes;
5401
        /* See note in ucnv_SBCSFromUTF8() about this goto. */
5402
0
    }
5403
5404
    /* conversion loop */
5405
0
    while(source<sourceLimit) {
5406
0
        if(targetCapacity>0) {
5407
0
            b=*source++;
5408
0
            if(U8_IS_SINGLE(b)) {
5409
                /* convert ASCII */
5410
0
                if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) {
5411
0
                    *target++=b;
5412
0
                    --targetCapacity;
5413
0
                    continue;
5414
0
                } else {
5415
0
                    value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, 0, b);
5416
0
                    if(value==0) {
5417
0
                        c=b;
5418
0
                        goto unassigned;
5419
0
                    }
5420
0
                }
5421
0
            } else {
5422
0
                if(b>=0xe0) {
5423
0
                    if( /* handle U+0800..U+D7FF inline */
5424
0
                        b<=0xed &&  // do not assume maxFastUChar>0xd7ff
5425
0
                        U8_IS_VALID_LEAD3_AND_T1(b, t1=source[0]) &&
5426
0
                        (t2=(uint8_t)(source[1]-0x80)) <= 0x3f
5427
0
                    ) {
5428
0
                        c=((b&0xf)<<6)|(t1&0x3f);
5429
0
                        source+=2;
5430
0
                        value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t2);
5431
0
                        if(value==0) {
5432
0
                            c=(c<<6)|t2;
5433
0
                            goto unassigned;
5434
0
                        }
5435
0
                    } else {
5436
0
                        c=-1;
5437
0
                    }
5438
0
                } else {
5439
0
                    if( /* handle U+0080..U+07FF inline */
5440
0
                        b>=0xc2 &&
5441
0
                        (t1=(uint8_t)(*source-0x80)) <= 0x3f
5442
0
                    ) {
5443
0
                        c=b&0x1f;
5444
0
                        ++source;
5445
0
                        value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t1);
5446
0
                        if(value==0) {
5447
0
                            c=(c<<6)|t1;
5448
0
                            goto unassigned;
5449
0
                        }
5450
0
                    } else {
5451
0
                        c=-1;
5452
0
                    }
5453
0
                }
5454
5455
0
                if(c<0) {
5456
                    /* handle "complicated" and error cases, and continuing partial characters */
5457
0
                    oldToULength=0;
5458
0
                    toULength=1;
5459
0
                    toULimit=U8_COUNT_BYTES_NON_ASCII(b);
5460
0
                    c=b;
5461
0
moreBytes:
5462
0
                    while(toULength<toULimit) {
5463
                        /*
5464
                         * The sourceLimit may have been adjusted before the conversion loop
5465
                         * to stop before a truncated sequence.
5466
                         * Here we need to use the real limit in case we have two truncated
5467
                         * sequences at the end.
5468
                         * See ticket #7492.
5469
                         */
5470
0
                        if(source<(uint8_t *)pToUArgs->sourceLimit) {
5471
0
                            b=*source;
5472
0
                            if(icu::UTF8::isValidTrail(c, b, toULength, toULimit)) {
5473
0
                                ++source;
5474
0
                                ++toULength;
5475
0
                                c=(c<<6)+b;
5476
0
                            } else {
5477
0
                                break; /* sequence too short, stop with toULength<toULimit */
5478
0
                            }
5479
0
                        } else {
5480
                            /* store the partial UTF-8 character, compatible with the regular UTF-8 converter */
5481
0
                            source-=(toULength-oldToULength);
5482
0
                            while(oldToULength<toULength) {
5483
0
                                utf8->toUBytes[oldToULength++]=*source++;
5484
0
                            }
5485
0
                            utf8->toUnicodeStatus=c;
5486
0
                            utf8->toULength=toULength;
5487
0
                            utf8->mode=toULimit;
5488
0
                            pToUArgs->source=(char *)source;
5489
0
                            pFromUArgs->target=(char *)target;
5490
0
                            return;
5491
0
                        }
5492
0
                    }
5493
5494
0
                    if(toULength==toULimit) {
5495
0
                        c-=utf8_offsets[toULength];
5496
0
                        if(toULength<=3) {  /* BMP */
5497
0
                            stage2Entry=MBCS_STAGE_2_FROM_U(table, c);
5498
0
                        } else {
5499
                            /* supplementary code point */
5500
0
                            if(!hasSupplementary) {
5501
                                /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
5502
0
                                stage2Entry=0;
5503
0
                            } else {
5504
0
                                stage2Entry=MBCS_STAGE_2_FROM_U(table, c);
5505
0
                            }
5506
0
                        }
5507
0
                    } else {
5508
                        /* error handling: illegal UTF-8 byte sequence */
5509
0
                        source-=(toULength-oldToULength);
5510
0
                        while(oldToULength<toULength) {
5511
0
                            utf8->toUBytes[oldToULength++]=*source++;
5512
0
                        }
5513
0
                        utf8->toULength=toULength;
5514
0
                        pToUArgs->source=(char *)source;
5515
0
                        pFromUArgs->target=(char *)target;
5516
0
                        *pErrorCode=U_ILLEGAL_CHAR_FOUND;
5517
0
                        return;
5518
0
                    }
5519
5520
                    /* get the bytes and the length for the output */
5521
                    /* MBCS_OUTPUT_2 */
5522
0
                    value=MBCS_VALUE_2_FROM_STAGE_2(results, stage2Entry, c);
5523
5524
                    /* is this code point assigned, or do we use fallbacks? */
5525
0
                    if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ||
5526
0
                         (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0))
5527
0
                    ) {
5528
0
                        goto unassigned;
5529
0
                    }
5530
0
                }
5531
0
            }
5532
5533
            /* write the output character bytes from value and length */
5534
            /* from the first if in the loop we know that targetCapacity>0 */
5535
0
            if(value<=0xff) {
5536
                /* this is easy because we know that there is enough space */
5537
0
                *target++=(uint8_t)value;
5538
0
                --targetCapacity;
5539
0
            } else /* length==2 */ {
5540
0
                *target++=(uint8_t)(value>>8);
5541
0
                if(2<=targetCapacity) {
5542
0
                    *target++=(uint8_t)value;
5543
0
                    targetCapacity-=2;
5544
0
                } else {
5545
0
                    cnv->charErrorBuffer[0]=(char)value;
5546
0
                    cnv->charErrorBufferLength=1;
5547
5548
                    /* target overflow */
5549
0
                    *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
5550
0
                    break;
5551
0
                }
5552
0
            }
5553
0
            continue;
5554
5555
0
unassigned:
5556
0
            {
5557
                /*
5558
                 * Try an extension mapping.
5559
                 * Pass in no source because we don't have UTF-16 input.
5560
                 * If we have a partial match on c, we will return and revert
5561
                 * to UTF-8->UTF-16->charset conversion.
5562
                 */
5563
0
                static const UChar nul=0;
5564
0
                const UChar *noSource=&nul;
5565
0
                c=_extFromU(cnv, cnv->sharedData,
5566
0
                            c, &noSource, noSource,
5567
0
                            &target, target+targetCapacity,
5568
0
                            NULL, -1,
5569
0
                            pFromUArgs->flush,
5570
0
                            pErrorCode);
5571
5572
0
                if(U_FAILURE(*pErrorCode)) {
5573
                    /* not mappable or buffer overflow */
5574
0
                    cnv->fromUChar32=c;
5575
0
                    break;
5576
0
                } else if(cnv->preFromUFirstCP>=0) {
5577
                    /*
5578
                     * Partial match, return and revert to pivoting.
5579
                     * In normal from-UTF-16 conversion, we would just continue
5580
                     * but then exit the loop because the extension match would
5581
                     * have consumed the source.
5582
                     */
5583
0
                    *pErrorCode=U_USING_DEFAULT_WARNING;
5584
0
                    break;
5585
0
                } else {
5586
                    /* a mapping was written to the target, continue */
5587
5588
                    /* recalculate the targetCapacity after an extension mapping */
5589
0
                    targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target);
5590
0
                    continue;
5591
0
                }
5592
0
            }
5593
0
        } else {
5594
            /* target is full */
5595
0
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
5596
0
            break;
5597
0
        }
5598
0
    }
5599
5600
    /*
5601
     * The sourceLimit may have been adjusted before the conversion loop
5602
     * to stop before a truncated sequence.
5603
     * If so, then collect the truncated sequence now.
5604
     */
5605
0
    if(U_SUCCESS(*pErrorCode) &&
5606
0
            cnv->preFromUFirstCP<0 &&
5607
0
            source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) {
5608
0
        c=utf8->toUBytes[0]=b=*source++;
5609
0
        toULength=1;
5610
0
        toULimit=U8_COUNT_BYTES(b);
5611
0
        while(source<sourceLimit) {
5612
0
            utf8->toUBytes[toULength++]=b=*source++;
5613
0
            c=(c<<6)+b;
5614
0
        }
5615
0
        utf8->toUnicodeStatus=c;
5616
0
        utf8->toULength=toULength;
5617
0
        utf8->mode=toULimit;
5618
0
    }
5619
5620
    /* write back the updated pointers */
5621
0
    pToUArgs->source=(char *)source;
5622
0
    pFromUArgs->target=(char *)target;
5623
0
}
5624
5625
/* miscellaneous ------------------------------------------------------------ */
5626
5627
static void U_CALLCONV
5628
ucnv_MBCSGetStarters(const UConverter* cnv,
5629
                 UBool starters[256],
5630
0
                 UErrorCode *) {
5631
0
    const int32_t *state0;
5632
0
    int i;
5633
5634
0
    state0=cnv->sharedData->mbcs.stateTable[cnv->sharedData->mbcs.dbcsOnlyState];
5635
0
    for(i=0; i<256; ++i) {
5636
        /* all bytes that cause a state transition from state 0 are lead bytes */
5637
0
        starters[i]= (UBool)MBCS_ENTRY_IS_TRANSITION(state0[i]);
5638
0
    }
5639
0
}
5640
5641
/*
5642
 * This is an internal function that allows other converter implementations
5643
 * to check whether a byte is a lead byte.
5644
 */
5645
U_CFUNC UBool
5646
0
ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte) {
5647
0
    return (UBool)MBCS_ENTRY_IS_TRANSITION(sharedData->mbcs.stateTable[0][(uint8_t)byte]);
5648
0
}
5649
5650
static void U_CALLCONV
5651
ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs,
5652
              int32_t offsetIndex,
5653
0
              UErrorCode *pErrorCode) {
5654
0
    UConverter *cnv=pArgs->converter;
5655
0
    char *p, *subchar;
5656
0
    char buffer[4];
5657
0
    int32_t length;
5658
5659
    /* first, select between subChar and subChar1 */
5660
0
    if( cnv->subChar1!=0 &&
5661
0
        (cnv->sharedData->mbcs.extIndexes!=NULL ?
5662
0
            cnv->useSubChar1 :
5663
0
            (cnv->invalidUCharBuffer[0]<=0xff))
5664
0
    ) {
5665
        /* select subChar1 if it is set (not 0) and the unmappable Unicode code point is up to U+00ff (IBM MBCS behavior) */
5666
0
        subchar=(char *)&cnv->subChar1;
5667
0
        length=1;
5668
0
    } else {
5669
        /* select subChar in all other cases */
5670
0
        subchar=(char *)cnv->subChars;
5671
0
        length=cnv->subCharLen;
5672
0
    }
5673
5674
    /* reset the selector for the next code point */
5675
0
    cnv->useSubChar1=FALSE;
5676
5677
0
    if (cnv->sharedData->mbcs.outputType == MBCS_OUTPUT_2_SISO) {
5678
0
        p=buffer;
5679
5680
        /* fromUnicodeStatus contains prevLength */
5681
0
        switch(length) {
5682
0
        case 1:
5683
0
            if(cnv->fromUnicodeStatus==2) {
5684
                /* DBCS mode and SBCS sub char: change to SBCS */
5685
0
                cnv->fromUnicodeStatus=1;
5686
0
                *p++=UCNV_SI;
5687
0
            }
5688
0
            *p++=subchar[0];
5689
0
            break;
5690
0
        case 2:
5691
0
            if(cnv->fromUnicodeStatus<=1) {
5692
                /* SBCS mode and DBCS sub char: change to DBCS */
5693
0
                cnv->fromUnicodeStatus=2;
5694
0
                *p++=UCNV_SO;
5695
0
            }
5696
0
            *p++=subchar[0];
5697
0
            *p++=subchar[1];
5698
0
            break;
5699
0
        default:
5700
0
            *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
5701
0
            return;
5702
0
        }
5703
0
        subchar=buffer;
5704
0
        length=(int32_t)(p-buffer);
5705
0
    }
5706
5707
0
    ucnv_cbFromUWriteBytes(pArgs, subchar, length, offsetIndex, pErrorCode);
5708
0
}
5709
5710
U_CFUNC UConverterType
5711
0
ucnv_MBCSGetType(const UConverter* converter) {
5712
    /* SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but here we cheat a little */
5713
0
    if(converter->sharedData->mbcs.countStates==1) {
5714
0
        return (UConverterType)UCNV_SBCS;
5715
0
    } else if((converter->sharedData->mbcs.outputType&0xff)==MBCS_OUTPUT_2_SISO) {
5716
0
        return (UConverterType)UCNV_EBCDIC_STATEFUL;
5717
0
    } else if(converter->sharedData->staticData->minBytesPerChar==2 && converter->sharedData->staticData->maxBytesPerChar==2) {
5718
0
        return (UConverterType)UCNV_DBCS;
5719
0
    }
5720
0
    return (UConverterType)UCNV_MBCS;
5721
0
}
5722
5723
#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */