/src/icu/source/common/ucnv_lmb.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // © 2016 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | /* |
4 | | ********************************************************************** |
5 | | * Copyright (C) 2000-2016, International Business Machines |
6 | | * Corporation and others. All Rights Reserved. |
7 | | ********************************************************************** |
8 | | * file name: ucnv_lmb.cpp |
9 | | * encoding: UTF-8 |
10 | | * tab size: 4 (not used) |
11 | | * indentation:4 |
12 | | * |
13 | | * created on: 2000feb09 |
14 | | * created by: Brendan Murray |
15 | | * extensively hacked up by: Jim Snyder-Grant |
16 | | * |
17 | | * Modification History: |
18 | | * |
19 | | * Date Name Description |
20 | | * |
21 | | * 06/20/2000 helena OS/400 port changes; mostly typecast. |
22 | | * 06/27/2000 Jim Snyder-Grant Deal with partial characters and small buffers. |
23 | | * Add comments to document LMBCS format and implementation |
24 | | * restructured order & breakdown of functions |
25 | | * 06/28/2000 helena Major rewrite for the callback API changes. |
26 | | */ |
27 | | |
28 | | #include "unicode/utypes.h" |
29 | | |
30 | | #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION && !UCONFIG_ONLY_HTML_CONVERSION |
31 | | |
32 | | #include "unicode/ucnv_err.h" |
33 | | #include "unicode/ucnv.h" |
34 | | #include "unicode/uset.h" |
35 | | #include "cmemory.h" |
36 | | #include "cstring.h" |
37 | | #include "uassert.h" |
38 | | #include "ucnv_imp.h" |
39 | | #include "ucnv_bld.h" |
40 | | #include "ucnv_cnv.h" |
41 | | |
42 | | #ifdef EBCDIC_RTL |
43 | | #include "ascii_a.h" |
44 | | #endif |
45 | | |
46 | | /* |
47 | | LMBCS |
48 | | |
49 | | (Lotus Multi-Byte Character Set) |
50 | | |
51 | | LMBCS was invented in the late 1980's and is primarily used in Lotus Notes |
52 | | databases and in Lotus 1-2-3 files. Programmers who work with the APIs |
53 | | into these products will sometimes need to deal with strings in this format. |
54 | | |
55 | | The code in this file provides an implementation for an ICU converter of |
56 | | LMBCS to and from Unicode. |
57 | | |
58 | | Since the LMBCS character set is only sparsely documented in existing |
59 | | printed or online material, we have added extensive annotation to this |
60 | | file to serve as a guide to understanding LMBCS. |
61 | | |
62 | | LMBCS was originally designed with these four sometimes-competing design goals: |
63 | | |
64 | | -Provide encodings for the characters in 12 existing national standards |
65 | | (plus a few other characters) |
66 | | -Minimal memory footprint |
67 | | -Maximal speed of conversion into the existing national character sets |
68 | | -No need to track a changing state as you interpret a string. |
69 | | |
70 | | |
71 | | All of the national character sets LMBCS was trying to encode are 'ANSI' |
72 | | based, in that the bytes from 0x20 - 0x7F are almost exactly the |
73 | | same common Latin unaccented characters and symbols in all character sets. |
74 | | |
75 | | So, in order to help meet the speed & memory design goals, the common ANSI |
76 | | bytes from 0x20-0x7F are represented by the same single-byte values in LMBCS. |
77 | | |
78 | | The general LMBCS code unit is from 1-3 bytes. We can describe the 3 bytes as |
79 | | follows: |
80 | | |
81 | | [G] D1 [D2] |
82 | | |
83 | | That is, a sometimes-optional 'group' byte, followed by 1 and sometimes 2 |
84 | | data bytes. The maximum size of a LMBCS character is 3 bytes: |
85 | | */ |
86 | | #define ULMBCS_CHARSIZE_MAX 3 |
87 | | /* |
88 | | The single-byte values from 0x20 to 0x7F are examples of single D1 bytes. |
89 | | We often have to figure out if byte values are below or above this, so we |
90 | | use the ANSI nomenclature 'C0' and 'C1' to refer to the range of control |
91 | | characters just above & below the common lower-ANSI range */ |
92 | 0 | #define ULMBCS_C0END 0x1F |
93 | 0 | #define ULMBCS_C1START 0x80 |
94 | | /* |
95 | | Since LMBCS is always dealing in byte units. we create a local type here for |
96 | | dealing with these units of LMBCS code units: |
97 | | |
98 | | */ |
99 | | typedef uint8_t ulmbcs_byte_t; |
100 | | |
101 | | /* |
102 | | Most of the values less than 0x20 are reserved in LMBCS to announce |
103 | | which national character standard is being used for the 'D' bytes. |
104 | | In the comments we show the common name and the IBM character-set ID |
105 | | for these character-set announcers: |
106 | | */ |
107 | | |
108 | 0 | #define ULMBCS_GRP_L1 0x01 /* Latin-1 :ibm-850 */ |
109 | | #define ULMBCS_GRP_GR 0x02 /* Greek :ibm-851 */ |
110 | | #define ULMBCS_GRP_HE 0x03 /* Hebrew :ibm-1255 */ |
111 | | #define ULMBCS_GRP_AR 0x04 /* Arabic :ibm-1256 */ |
112 | | #define ULMBCS_GRP_RU 0x05 /* Cyrillic :ibm-1251 */ |
113 | | #define ULMBCS_GRP_L2 0x06 /* Latin-2 :ibm-852 */ |
114 | | #define ULMBCS_GRP_TR 0x08 /* Turkish :ibm-1254 */ |
115 | 0 | #define ULMBCS_GRP_TH 0x0B /* Thai :ibm-874 */ |
116 | | #define ULMBCS_GRP_JA 0x10 /* Japanese :ibm-943 */ |
117 | | #define ULMBCS_GRP_KO 0x11 /* Korean :ibm-1261 */ |
118 | | #define ULMBCS_GRP_TW 0x12 /* Chinese SC :ibm-950 */ |
119 | | #define ULMBCS_GRP_CN 0x13 /* Chinese TC :ibm-1386 */ |
120 | | |
121 | | /* |
122 | | So, the beginning of understanding LMBCS is that IF the first byte of a LMBCS |
123 | | character is one of those 12 values, you can interpret the remaining bytes of |
124 | | that character as coming from one of those character sets. Since the lower |
125 | | ANSI bytes already are represented in single bytes, using one of the character |
126 | | set announcers is used to announce a character that starts with a byte of |
127 | | 0x80 or greater. |
128 | | |
129 | | The character sets are arranged so that the single byte sets all appear |
130 | | before the multi-byte character sets. When we need to tell whether a |
131 | | group byte is for a single byte char set or not we use this define: */ |
132 | | |
133 | 0 | #define ULMBCS_DOUBLEOPTGROUP_START 0x10 |
134 | | |
135 | | /* |
136 | | However, to fully understand LMBCS, you must also understand a series of |
137 | | exceptions & optimizations made in service of the design goals. |
138 | | |
139 | | First, those of you who are character set mavens may have noticed that |
140 | | the 'double-byte' character sets are actually multi-byte character sets |
141 | | that can have 1 or two bytes, even in the upper-ascii range. To force |
142 | | each group byte to introduce a fixed-width encoding (to make it faster to |
143 | | count characters), we use a convention of doubling up on the group byte |
144 | | to introduce any single-byte character > 0x80 in an otherwise double-byte |
145 | | character set. So, for example, the LMBCS sequence x10 x10 xAE is the |
146 | | same as '0xAE' in the Japanese code page 943. |
147 | | |
148 | | Next, you will notice that the list of group bytes has some gaps. |
149 | | These are used in various ways. |
150 | | |
151 | | We reserve a few special single byte values for common control |
152 | | characters. These are in the same place as their ANSI equivalents for speed. |
153 | | */ |
154 | | |
155 | 0 | #define ULMBCS_HT 0x09 /* Fixed control char - Horizontal Tab */ |
156 | 0 | #define ULMBCS_LF 0x0A /* Fixed control char - Line Feed */ |
157 | 0 | #define ULMBCS_CR 0x0D /* Fixed control char - Carriage Return */ |
158 | | |
159 | | /* Then, 1-2-3 reserved a special single-byte character to put at the |
160 | | beginning of internal 'system' range names: */ |
161 | | |
162 | 0 | #define ULMBCS_123SYSTEMRANGE 0x19 |
163 | | |
164 | | /* Then we needed a place to put all the other ansi control characters |
165 | | that must be moved to different values because LMBCS reserves those |
166 | | values for other purposes. To represent the control characters, we start |
167 | | with a first byte of 0xF & add the control character value as the |
168 | | second byte */ |
169 | 0 | #define ULMBCS_GRP_CTRL 0x0F |
170 | | |
171 | | /* For the C0 controls (less than 0x20), we add 0x20 to preserve the |
172 | | useful doctrine that any byte less than 0x20 in a LMBCS char must be |
173 | | the first byte of a character:*/ |
174 | 0 | #define ULMBCS_CTRLOFFSET 0x20 |
175 | | |
176 | | /* |
177 | | Where to put the characters that aren't part of any of the 12 national |
178 | | character sets? The first thing that was done, in the earlier years of |
179 | | LMBCS, was to use up the spaces of the form |
180 | | |
181 | | [G] D1, |
182 | | |
183 | | where 'G' was one of the single-byte character groups, and |
184 | | D1 was less than 0x80. These sequences are gathered together |
185 | | into a Lotus-invented doublebyte character set to represent a |
186 | | lot of stray values. Internally, in this implementation, we track this |
187 | | as group '0', as a place to tuck this exceptions list.*/ |
188 | | |
189 | 0 | #define ULMBCS_GRP_EXCEPT 0x00 |
190 | | /* |
191 | | Finally, as the durability and usefulness of UNICODE became clear, |
192 | | LOTUS added a new group 0x14 to hold Unicode values not otherwise |
193 | | represented in LMBCS: */ |
194 | 0 | #define ULMBCS_GRP_UNICODE 0x14 |
195 | | /* The two bytes appearing after a 0x14 are interpreted as UFT-16 BE |
196 | | (Big-Endian) characters. The exception comes when the UTF16 |
197 | | representation would have a zero as the second byte. In that case, |
198 | | 'F6' is used in its place, and the bytes are swapped. (This prevents |
199 | | LMBCS from encoding any Unicode values of the form U+F6xx, but that's OK: |
200 | | 0xF6xx is in the middle of the Private Use Area.)*/ |
201 | 0 | #define ULMBCS_UNICOMPATZERO 0xF6 |
202 | | |
203 | | /* It is also useful in our code to have a constant for the size of |
204 | | a LMBCS char that holds a literal Unicode value */ |
205 | 0 | #define ULMBCS_UNICODE_SIZE 3 |
206 | | |
207 | | /* |
208 | | To squish the LMBCS representations down even further, and to make |
209 | | translations even faster,sometimes the optimization group byte can be dropped |
210 | | from a LMBCS character. This is decided on a process-by-process basis. The |
211 | | group byte that is dropped is called the 'optimization group'. |
212 | | |
213 | | For Notes, the optimzation group is always 0x1.*/ |
214 | | #define ULMBCS_DEFAULTOPTGROUP 0x1 |
215 | | /* For 1-2-3 files, the optimzation group is stored in the header of the 1-2-3 |
216 | | file. |
217 | | |
218 | | In any case, when using ICU, you either pass in the |
219 | | optimization group as part of the name of the converter (LMBCS-1, LMBCS-2, |
220 | | etc.). Using plain 'LMBCS' as the name of the converter will give you |
221 | | LMBCS-1. |
222 | | |
223 | | |
224 | | *** Implementation strategy *** |
225 | | |
226 | | |
227 | | Because of the extensive use of other character sets, the LMBCS converter |
228 | | keeps a mapping between optimization groups and IBM character sets, so that |
229 | | ICU converters can be created and used as needed. */ |
230 | | |
231 | | /* As you can see, even though any byte below 0x20 could be an optimization |
232 | | byte, only those at 0x13 or below can map to an actual converter. To limit |
233 | | some loops and searches, we define a value for that last group converter:*/ |
234 | | |
235 | 0 | #define ULMBCS_GRP_LAST 0x13 /* last LMBCS group that has a converter */ |
236 | | |
237 | | static const char * const OptGroupByteToCPName[ULMBCS_GRP_LAST + 1] = { |
238 | | /* 0x0000 */ "lmb-excp", /* internal home for the LOTUS exceptions list */ |
239 | | /* 0x0001 */ "ibm-850", |
240 | | /* 0x0002 */ "ibm-851", |
241 | | /* 0x0003 */ "windows-1255", |
242 | | /* 0x0004 */ "windows-1256", |
243 | | /* 0x0005 */ "windows-1251", |
244 | | /* 0x0006 */ "ibm-852", |
245 | | /* 0x0007 */ NULL, /* Unused */ |
246 | | /* 0x0008 */ "windows-1254", |
247 | | /* 0x0009 */ NULL, /* Control char HT */ |
248 | | /* 0x000A */ NULL, /* Control char LF */ |
249 | | /* 0x000B */ "windows-874", |
250 | | /* 0x000C */ NULL, /* Unused */ |
251 | | /* 0x000D */ NULL, /* Control char CR */ |
252 | | /* 0x000E */ NULL, /* Unused */ |
253 | | /* 0x000F */ NULL, /* Control chars: 0x0F20 + C0/C1 character: algorithmic */ |
254 | | /* 0x0010 */ "windows-932", |
255 | | /* 0x0011 */ "windows-949", |
256 | | /* 0x0012 */ "windows-950", |
257 | | /* 0x0013 */ "windows-936" |
258 | | |
259 | | /* The rest are null, including the 0x0014 Unicode compatibility region |
260 | | and 0x0019, the 1-2-3 system range control char */ |
261 | | }; |
262 | | |
263 | | |
264 | | /* That's approximately all the data that's needed for translating |
265 | | LMBCS to Unicode. |
266 | | |
267 | | |
268 | | However, to translate Unicode to LMBCS, we need some more support. |
269 | | |
270 | | That's because there are often more than one possible mappings from a Unicode |
271 | | code point back into LMBCS. The first thing we do is look up into a table |
272 | | to figure out if there are more than one possible mappings. This table, |
273 | | arranged by Unicode values (including ranges) either lists which group |
274 | | to use, or says that it could go into one or more of the SBCS sets, or |
275 | | into one or more of the DBCS sets. (If the character exists in both DBCS & |
276 | | SBCS, the table will place it in the SBCS sets, to make the LMBCS code point |
277 | | length as small as possible. Here's the two special markers we use to indicate |
278 | | ambiguous mappings: */ |
279 | | |
280 | 0 | #define ULMBCS_AMBIGUOUS_SBCS 0x80 /* could fit in more than one |
281 | | LMBCS sbcs native encoding |
282 | | (example: most accented latin) */ |
283 | 0 | #define ULMBCS_AMBIGUOUS_MBCS 0x81 /* could fit in more than one |
284 | | LMBCS mbcs native encoding |
285 | | (example: Unihan) */ |
286 | 0 | #define ULMBCS_AMBIGUOUS_ALL 0x82 |
287 | | /* And here's a simple way to see if a group falls in an appropriate range */ |
288 | | #define ULMBCS_AMBIGUOUS_MATCH(agroup, xgroup) \ |
289 | 0 | ((((agroup) == ULMBCS_AMBIGUOUS_SBCS) && \ |
290 | 0 | (xgroup) < ULMBCS_DOUBLEOPTGROUP_START) || \ |
291 | 0 | (((agroup) == ULMBCS_AMBIGUOUS_MBCS) && \ |
292 | 0 | (xgroup) >= ULMBCS_DOUBLEOPTGROUP_START)) || \ |
293 | 0 | ((agroup) == ULMBCS_AMBIGUOUS_ALL) |
294 | | |
295 | | |
296 | | /* The table & some code to use it: */ |
297 | | |
298 | | |
299 | | static const struct _UniLMBCSGrpMap |
300 | | { |
301 | | const UChar uniStartRange; |
302 | | const UChar uniEndRange; |
303 | | const ulmbcs_byte_t GrpType; |
304 | | } UniLMBCSGrpMap[] |
305 | | = |
306 | | { |
307 | | |
308 | | {0x0001, 0x001F, ULMBCS_GRP_CTRL}, |
309 | | {0x0080, 0x009F, ULMBCS_GRP_CTRL}, |
310 | | {0x00A0, 0x00A6, ULMBCS_AMBIGUOUS_SBCS}, |
311 | | {0x00A7, 0x00A8, ULMBCS_AMBIGUOUS_ALL}, |
312 | | {0x00A9, 0x00AF, ULMBCS_AMBIGUOUS_SBCS}, |
313 | | {0x00B0, 0x00B1, ULMBCS_AMBIGUOUS_ALL}, |
314 | | {0x00B2, 0x00B3, ULMBCS_AMBIGUOUS_SBCS}, |
315 | | {0x00B4, 0x00B4, ULMBCS_AMBIGUOUS_ALL}, |
316 | | {0x00B5, 0x00B5, ULMBCS_AMBIGUOUS_SBCS}, |
317 | | {0x00B6, 0x00B6, ULMBCS_AMBIGUOUS_ALL}, |
318 | | {0x00B7, 0x00D6, ULMBCS_AMBIGUOUS_SBCS}, |
319 | | {0x00D7, 0x00D7, ULMBCS_AMBIGUOUS_ALL}, |
320 | | {0x00D8, 0x00F6, ULMBCS_AMBIGUOUS_SBCS}, |
321 | | {0x00F7, 0x00F7, ULMBCS_AMBIGUOUS_ALL}, |
322 | | {0x00F8, 0x01CD, ULMBCS_AMBIGUOUS_SBCS}, |
323 | | {0x01CE, 0x01CE, ULMBCS_GRP_TW }, |
324 | | {0x01CF, 0x02B9, ULMBCS_AMBIGUOUS_SBCS}, |
325 | | {0x02BA, 0x02BA, ULMBCS_GRP_CN}, |
326 | | {0x02BC, 0x02C8, ULMBCS_AMBIGUOUS_SBCS}, |
327 | | {0x02C9, 0x02D0, ULMBCS_AMBIGUOUS_MBCS}, |
328 | | {0x02D8, 0x02DD, ULMBCS_AMBIGUOUS_SBCS}, |
329 | | {0x0384, 0x0390, ULMBCS_AMBIGUOUS_SBCS}, |
330 | | {0x0391, 0x03A9, ULMBCS_AMBIGUOUS_ALL}, |
331 | | {0x03AA, 0x03B0, ULMBCS_AMBIGUOUS_SBCS}, |
332 | | {0x03B1, 0x03C9, ULMBCS_AMBIGUOUS_ALL}, |
333 | | {0x03CA, 0x03CE, ULMBCS_AMBIGUOUS_SBCS}, |
334 | | {0x0400, 0x0400, ULMBCS_GRP_RU}, |
335 | | {0x0401, 0x0401, ULMBCS_AMBIGUOUS_ALL}, |
336 | | {0x0402, 0x040F, ULMBCS_GRP_RU}, |
337 | | {0x0410, 0x0431, ULMBCS_AMBIGUOUS_ALL}, |
338 | | {0x0432, 0x044E, ULMBCS_GRP_RU}, |
339 | | {0x044F, 0x044F, ULMBCS_AMBIGUOUS_ALL}, |
340 | | {0x0450, 0x0491, ULMBCS_GRP_RU}, |
341 | | {0x05B0, 0x05F2, ULMBCS_GRP_HE}, |
342 | | {0x060C, 0x06AF, ULMBCS_GRP_AR}, |
343 | | {0x0E01, 0x0E5B, ULMBCS_GRP_TH}, |
344 | | {0x200C, 0x200F, ULMBCS_AMBIGUOUS_SBCS}, |
345 | | {0x2010, 0x2010, ULMBCS_AMBIGUOUS_MBCS}, |
346 | | {0x2013, 0x2014, ULMBCS_AMBIGUOUS_SBCS}, |
347 | | {0x2015, 0x2015, ULMBCS_AMBIGUOUS_MBCS}, |
348 | | {0x2016, 0x2016, ULMBCS_AMBIGUOUS_MBCS}, |
349 | | {0x2017, 0x2017, ULMBCS_AMBIGUOUS_SBCS}, |
350 | | {0x2018, 0x2019, ULMBCS_AMBIGUOUS_ALL}, |
351 | | {0x201A, 0x201B, ULMBCS_AMBIGUOUS_SBCS}, |
352 | | {0x201C, 0x201D, ULMBCS_AMBIGUOUS_ALL}, |
353 | | {0x201E, 0x201F, ULMBCS_AMBIGUOUS_SBCS}, |
354 | | {0x2020, 0x2021, ULMBCS_AMBIGUOUS_ALL}, |
355 | | {0x2022, 0x2024, ULMBCS_AMBIGUOUS_SBCS}, |
356 | | {0x2025, 0x2025, ULMBCS_AMBIGUOUS_MBCS}, |
357 | | {0x2026, 0x2026, ULMBCS_AMBIGUOUS_ALL}, |
358 | | {0x2027, 0x2027, ULMBCS_GRP_TW}, |
359 | | {0x2030, 0x2030, ULMBCS_AMBIGUOUS_ALL}, |
360 | | {0x2031, 0x2031, ULMBCS_AMBIGUOUS_SBCS}, |
361 | | {0x2032, 0x2033, ULMBCS_AMBIGUOUS_MBCS}, |
362 | | {0x2035, 0x2035, ULMBCS_AMBIGUOUS_MBCS}, |
363 | | {0x2039, 0x203A, ULMBCS_AMBIGUOUS_SBCS}, |
364 | | {0x203B, 0x203B, ULMBCS_AMBIGUOUS_MBCS}, |
365 | | {0x203C, 0x203C, ULMBCS_GRP_EXCEPT}, |
366 | | {0x2074, 0x2074, ULMBCS_GRP_KO}, |
367 | | {0x207F, 0x207F, ULMBCS_GRP_EXCEPT}, |
368 | | {0x2081, 0x2084, ULMBCS_GRP_KO}, |
369 | | {0x20A4, 0x20AC, ULMBCS_AMBIGUOUS_SBCS}, |
370 | | {0x2103, 0x2109, ULMBCS_AMBIGUOUS_MBCS}, |
371 | | {0x2111, 0x2120, ULMBCS_AMBIGUOUS_SBCS}, |
372 | | /*zhujin: upgrade, for regressiont test, spr HKIA4YHTSU*/ |
373 | | {0x2121, 0x2121, ULMBCS_AMBIGUOUS_MBCS}, |
374 | | {0x2122, 0x2126, ULMBCS_AMBIGUOUS_SBCS}, |
375 | | {0x212B, 0x212B, ULMBCS_AMBIGUOUS_MBCS}, |
376 | | {0x2135, 0x2135, ULMBCS_AMBIGUOUS_SBCS}, |
377 | | {0x2153, 0x2154, ULMBCS_GRP_KO}, |
378 | | {0x215B, 0x215E, ULMBCS_GRP_EXCEPT}, |
379 | | {0x2160, 0x2179, ULMBCS_AMBIGUOUS_MBCS}, |
380 | | {0x2190, 0x2193, ULMBCS_AMBIGUOUS_ALL}, |
381 | | {0x2194, 0x2195, ULMBCS_GRP_EXCEPT}, |
382 | | {0x2196, 0x2199, ULMBCS_AMBIGUOUS_MBCS}, |
383 | | {0x21A8, 0x21A8, ULMBCS_GRP_EXCEPT}, |
384 | | {0x21B8, 0x21B9, ULMBCS_GRP_CN}, |
385 | | {0x21D0, 0x21D1, ULMBCS_GRP_EXCEPT}, |
386 | | {0x21D2, 0x21D2, ULMBCS_AMBIGUOUS_MBCS}, |
387 | | {0x21D3, 0x21D3, ULMBCS_GRP_EXCEPT}, |
388 | | {0x21D4, 0x21D4, ULMBCS_AMBIGUOUS_MBCS}, |
389 | | {0x21D5, 0x21D5, ULMBCS_GRP_EXCEPT}, |
390 | | {0x21E7, 0x21E7, ULMBCS_GRP_CN}, |
391 | | {0x2200, 0x2200, ULMBCS_AMBIGUOUS_MBCS}, |
392 | | {0x2201, 0x2201, ULMBCS_GRP_EXCEPT}, |
393 | | {0x2202, 0x2202, ULMBCS_AMBIGUOUS_MBCS}, |
394 | | {0x2203, 0x2203, ULMBCS_AMBIGUOUS_MBCS}, |
395 | | {0x2204, 0x2206, ULMBCS_GRP_EXCEPT}, |
396 | | {0x2207, 0x2208, ULMBCS_AMBIGUOUS_MBCS}, |
397 | | {0x2209, 0x220A, ULMBCS_GRP_EXCEPT}, |
398 | | {0x220B, 0x220B, ULMBCS_AMBIGUOUS_MBCS}, |
399 | | {0x220F, 0x2215, ULMBCS_AMBIGUOUS_MBCS}, |
400 | | {0x2219, 0x2219, ULMBCS_GRP_EXCEPT}, |
401 | | {0x221A, 0x221A, ULMBCS_AMBIGUOUS_MBCS}, |
402 | | {0x221B, 0x221C, ULMBCS_GRP_EXCEPT}, |
403 | | {0x221D, 0x221E, ULMBCS_AMBIGUOUS_MBCS}, |
404 | | {0x221F, 0x221F, ULMBCS_GRP_EXCEPT}, |
405 | | {0x2220, 0x2220, ULMBCS_AMBIGUOUS_MBCS}, |
406 | | {0x2223, 0x222A, ULMBCS_AMBIGUOUS_MBCS}, |
407 | | {0x222B, 0x223D, ULMBCS_AMBIGUOUS_MBCS}, |
408 | | {0x2245, 0x2248, ULMBCS_GRP_EXCEPT}, |
409 | | {0x224C, 0x224C, ULMBCS_GRP_TW}, |
410 | | {0x2252, 0x2252, ULMBCS_AMBIGUOUS_MBCS}, |
411 | | {0x2260, 0x2261, ULMBCS_AMBIGUOUS_MBCS}, |
412 | | {0x2262, 0x2265, ULMBCS_GRP_EXCEPT}, |
413 | | {0x2266, 0x226F, ULMBCS_AMBIGUOUS_MBCS}, |
414 | | {0x2282, 0x2283, ULMBCS_AMBIGUOUS_MBCS}, |
415 | | {0x2284, 0x2285, ULMBCS_GRP_EXCEPT}, |
416 | | {0x2286, 0x2287, ULMBCS_AMBIGUOUS_MBCS}, |
417 | | {0x2288, 0x2297, ULMBCS_GRP_EXCEPT}, |
418 | | {0x2299, 0x22BF, ULMBCS_AMBIGUOUS_MBCS}, |
419 | | {0x22C0, 0x22C0, ULMBCS_GRP_EXCEPT}, |
420 | | {0x2310, 0x2310, ULMBCS_GRP_EXCEPT}, |
421 | | {0x2312, 0x2312, ULMBCS_AMBIGUOUS_MBCS}, |
422 | | {0x2318, 0x2321, ULMBCS_GRP_EXCEPT}, |
423 | | {0x2318, 0x2321, ULMBCS_GRP_CN}, |
424 | | {0x2460, 0x24E9, ULMBCS_AMBIGUOUS_MBCS}, |
425 | | {0x2500, 0x2500, ULMBCS_AMBIGUOUS_SBCS}, |
426 | | {0x2501, 0x2501, ULMBCS_AMBIGUOUS_MBCS}, |
427 | | {0x2502, 0x2502, ULMBCS_AMBIGUOUS_ALL}, |
428 | | {0x2503, 0x2503, ULMBCS_AMBIGUOUS_MBCS}, |
429 | | {0x2504, 0x2505, ULMBCS_GRP_TW}, |
430 | | {0x2506, 0x2665, ULMBCS_AMBIGUOUS_ALL}, |
431 | | {0x2666, 0x2666, ULMBCS_GRP_EXCEPT}, |
432 | | {0x2667, 0x2669, ULMBCS_AMBIGUOUS_SBCS}, |
433 | | {0x266A, 0x266A, ULMBCS_AMBIGUOUS_ALL}, |
434 | | {0x266B, 0x266C, ULMBCS_AMBIGUOUS_SBCS}, |
435 | | {0x266D, 0x266D, ULMBCS_AMBIGUOUS_MBCS}, |
436 | | {0x266E, 0x266E, ULMBCS_AMBIGUOUS_SBCS}, |
437 | | {0x266F, 0x266F, ULMBCS_GRP_JA}, |
438 | | {0x2670, 0x2E7F, ULMBCS_AMBIGUOUS_SBCS}, |
439 | | {0x2E80, 0xF861, ULMBCS_AMBIGUOUS_MBCS}, |
440 | | {0xF862, 0xF8FF, ULMBCS_GRP_EXCEPT}, |
441 | | {0xF900, 0xFA2D, ULMBCS_AMBIGUOUS_MBCS}, |
442 | | {0xFB00, 0xFEFF, ULMBCS_AMBIGUOUS_SBCS}, |
443 | | {0xFF01, 0xFFEE, ULMBCS_AMBIGUOUS_MBCS}, |
444 | | {0xFFFF, 0xFFFF, ULMBCS_GRP_UNICODE} |
445 | | }; |
446 | | |
447 | | static ulmbcs_byte_t |
448 | | FindLMBCSUniRange(UChar uniChar) |
449 | 0 | { |
450 | 0 | const struct _UniLMBCSGrpMap * pTable = UniLMBCSGrpMap; |
451 | |
|
452 | 0 | while (uniChar > pTable->uniEndRange) |
453 | 0 | { |
454 | 0 | pTable++; |
455 | 0 | } |
456 | |
|
457 | 0 | if (uniChar >= pTable->uniStartRange) |
458 | 0 | { |
459 | 0 | return pTable->GrpType; |
460 | 0 | } |
461 | 0 | return ULMBCS_GRP_UNICODE; |
462 | 0 | } |
463 | | |
464 | | /* |
465 | | We also ask the creator of a converter to send in a preferred locale |
466 | | that we can use in resolving ambiguous mappings. They send the locale |
467 | | in as a string, and we map it, if possible, to one of the |
468 | | LMBCS groups. We use this table, and the associated code, to |
469 | | do the lookup: */ |
470 | | |
471 | | /************************************************** |
472 | | This table maps locale ID's to LMBCS opt groups. |
473 | | The default return is group 0x01. Note that for |
474 | | performance reasons, the table is sorted in |
475 | | increasing alphabetic order, with the notable |
476 | | exception of zhTW. This is to force the check |
477 | | for Traditonal Chinese before dropping back to |
478 | | Simplified. |
479 | | |
480 | | Note too that the Latin-1 groups have been |
481 | | commented out because it's the default, and |
482 | | this shortens the table, allowing a serial |
483 | | search to go quickly. |
484 | | *************************************************/ |
485 | | |
486 | | static const struct _LocaleLMBCSGrpMap |
487 | | { |
488 | | const char *LocaleID; |
489 | | const ulmbcs_byte_t OptGroup; |
490 | | } LocaleLMBCSGrpMap[] = |
491 | | { |
492 | | {"ar", ULMBCS_GRP_AR}, |
493 | | {"be", ULMBCS_GRP_RU}, |
494 | | {"bg", ULMBCS_GRP_L2}, |
495 | | /* {"ca", ULMBCS_GRP_L1}, */ |
496 | | {"cs", ULMBCS_GRP_L2}, |
497 | | /* {"da", ULMBCS_GRP_L1}, */ |
498 | | /* {"de", ULMBCS_GRP_L1}, */ |
499 | | {"el", ULMBCS_GRP_GR}, |
500 | | /* {"en", ULMBCS_GRP_L1}, */ |
501 | | /* {"es", ULMBCS_GRP_L1}, */ |
502 | | /* {"et", ULMBCS_GRP_L1}, */ |
503 | | /* {"fi", ULMBCS_GRP_L1}, */ |
504 | | /* {"fr", ULMBCS_GRP_L1}, */ |
505 | | {"he", ULMBCS_GRP_HE}, |
506 | | {"hu", ULMBCS_GRP_L2}, |
507 | | /* {"is", ULMBCS_GRP_L1}, */ |
508 | | /* {"it", ULMBCS_GRP_L1}, */ |
509 | | {"iw", ULMBCS_GRP_HE}, |
510 | | {"ja", ULMBCS_GRP_JA}, |
511 | | {"ko", ULMBCS_GRP_KO}, |
512 | | /* {"lt", ULMBCS_GRP_L1}, */ |
513 | | /* {"lv", ULMBCS_GRP_L1}, */ |
514 | | {"mk", ULMBCS_GRP_RU}, |
515 | | /* {"nl", ULMBCS_GRP_L1}, */ |
516 | | /* {"no", ULMBCS_GRP_L1}, */ |
517 | | {"pl", ULMBCS_GRP_L2}, |
518 | | /* {"pt", ULMBCS_GRP_L1}, */ |
519 | | {"ro", ULMBCS_GRP_L2}, |
520 | | {"ru", ULMBCS_GRP_RU}, |
521 | | {"sh", ULMBCS_GRP_L2}, |
522 | | {"sk", ULMBCS_GRP_L2}, |
523 | | {"sl", ULMBCS_GRP_L2}, |
524 | | {"sq", ULMBCS_GRP_L2}, |
525 | | {"sr", ULMBCS_GRP_RU}, |
526 | | /* {"sv", ULMBCS_GRP_L1}, */ |
527 | | {"th", ULMBCS_GRP_TH}, |
528 | | {"tr", ULMBCS_GRP_TR}, |
529 | | {"uk", ULMBCS_GRP_RU}, |
530 | | /* {"vi", ULMBCS_GRP_L1}, */ |
531 | | {"zhTW", ULMBCS_GRP_TW}, |
532 | | {"zh", ULMBCS_GRP_CN}, |
533 | | {NULL, ULMBCS_GRP_L1} |
534 | | }; |
535 | | |
536 | | |
537 | | static ulmbcs_byte_t |
538 | | FindLMBCSLocale(const char *LocaleID) |
539 | 0 | { |
540 | 0 | const struct _LocaleLMBCSGrpMap *pTable = LocaleLMBCSGrpMap; |
541 | |
|
542 | 0 | if ((!LocaleID) || (!*LocaleID)) |
543 | 0 | { |
544 | 0 | return 0; |
545 | 0 | } |
546 | | |
547 | 0 | while (pTable->LocaleID) |
548 | 0 | { |
549 | 0 | if (*pTable->LocaleID == *LocaleID) /* Check only first char for speed */ |
550 | 0 | { |
551 | | /* First char matches - check whole name, for entry-length */ |
552 | 0 | if (uprv_strncmp(pTable->LocaleID, LocaleID, strlen(pTable->LocaleID)) == 0) |
553 | 0 | return pTable->OptGroup; |
554 | 0 | } |
555 | 0 | else |
556 | 0 | if (*pTable->LocaleID > *LocaleID) /* Sorted alphabetically - exit */ |
557 | 0 | break; |
558 | 0 | pTable++; |
559 | 0 | } |
560 | 0 | return ULMBCS_GRP_L1; |
561 | 0 | } |
562 | | |
563 | | |
564 | | /* |
565 | | Before we get to the main body of code, here's how we hook up to the rest |
566 | | of ICU. ICU converters are required to define a structure that includes |
567 | | some function pointers, and some common data, in the style of a C++ |
568 | | vtable. There is also room in there for converter-specific data. LMBCS |
569 | | uses that converter-specific data to keep track of the 12 subconverters |
570 | | we use, the optimization group, and the group (if any) that matches the |
571 | | locale. We have one structure instantiated for each of the 12 possible |
572 | | optimization groups. To avoid typos & to avoid boring the reader, we |
573 | | put the declarations of these structures and functions into macros. To see |
574 | | the definitions of these structures, see unicode\ucnv_bld.h |
575 | | */ |
576 | | |
577 | | typedef struct |
578 | | { |
579 | | UConverterSharedData *OptGrpConverter[ULMBCS_GRP_LAST+1]; /* Converter per Opt. grp. */ |
580 | | uint8_t OptGroup; /* default Opt. grp. for this LMBCS session */ |
581 | | uint8_t localeConverterIndex; /* reasonable locale match for index */ |
582 | | } |
583 | | UConverterDataLMBCS; |
584 | | |
585 | | U_CDECL_BEGIN |
586 | | static void U_CALLCONV _LMBCSClose(UConverter * _this); |
587 | | U_CDECL_END |
588 | | |
589 | | #define DECLARE_LMBCS_DATA(n) \ |
590 | | static const UConverterImpl _LMBCSImpl##n={\ |
591 | | UCNV_LMBCS_##n,\ |
592 | | NULL,NULL,\ |
593 | | _LMBCSOpen##n,\ |
594 | | _LMBCSClose,\ |
595 | | NULL,\ |
596 | | _LMBCSToUnicodeWithOffsets,\ |
597 | | _LMBCSToUnicodeWithOffsets,\ |
598 | | _LMBCSFromUnicode,\ |
599 | | _LMBCSFromUnicode,\ |
600 | | NULL,\ |
601 | | NULL,\ |
602 | | NULL,\ |
603 | | NULL,\ |
604 | | _LMBCSSafeClone,\ |
605 | | ucnv_getCompleteUnicodeSet,\ |
606 | | NULL,\ |
607 | | NULL\ |
608 | | };\ |
609 | | static const UConverterStaticData _LMBCSStaticData##n={\ |
610 | | sizeof(UConverterStaticData),\ |
611 | | "LMBCS-" #n,\ |
612 | | 0, UCNV_IBM, UCNV_LMBCS_##n, 1, 3,\ |
613 | | { 0x3f, 0, 0, 0 },1,FALSE,FALSE,0,0,{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} \ |
614 | | };\ |
615 | | const UConverterSharedData _LMBCSData##n= \ |
616 | | UCNV_IMMUTABLE_SHARED_DATA_INITIALIZER(&_LMBCSStaticData##n, &_LMBCSImpl##n); |
617 | | |
618 | | /* The only function we needed to duplicate 12 times was the 'open' |
619 | | function, which will do basically the same thing except set a different |
620 | | optimization group. So, we put the common stuff into a worker function, |
621 | | and set up another macro to stamp out the 12 open functions:*/ |
622 | | #define DEFINE_LMBCS_OPEN(n) \ |
623 | | static void U_CALLCONV \ |
624 | 0 | _LMBCSOpen##n(UConverter* _this, UConverterLoadArgs* pArgs, UErrorCode* err) \ |
625 | 0 | { _LMBCSOpenWorker(_this, pArgs, err, n); } Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen1(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen2(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen3(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen4(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen5(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen6(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen8(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen11(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen16(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen17(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen18(UConverter*, UConverterLoadArgs*, UErrorCode*) Unexecuted instantiation: ucnv_lmb.cpp:_LMBCSOpen19(UConverter*, UConverterLoadArgs*, UErrorCode*) |
626 | | |
627 | | |
628 | | |
629 | | /* Here's the open worker & the common close function */ |
630 | | static void |
631 | | _LMBCSOpenWorker(UConverter* _this, |
632 | | UConverterLoadArgs *pArgs, |
633 | | UErrorCode* err, |
634 | | ulmbcs_byte_t OptGroup) |
635 | 0 | { |
636 | 0 | UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS*)uprv_malloc (sizeof (UConverterDataLMBCS)); |
637 | 0 | _this->extraInfo = extraInfo; |
638 | 0 | if(extraInfo != NULL) |
639 | 0 | { |
640 | 0 | UConverterNamePieces stackPieces; |
641 | 0 | UConverterLoadArgs stackArgs= UCNV_LOAD_ARGS_INITIALIZER; |
642 | 0 | ulmbcs_byte_t i; |
643 | |
|
644 | 0 | uprv_memset(extraInfo, 0, sizeof(UConverterDataLMBCS)); |
645 | |
|
646 | 0 | stackArgs.onlyTestIsLoadable = pArgs->onlyTestIsLoadable; |
647 | |
|
648 | 0 | for (i=0; i <= ULMBCS_GRP_LAST && U_SUCCESS(*err); i++) |
649 | 0 | { |
650 | 0 | if(OptGroupByteToCPName[i] != NULL) { |
651 | 0 | extraInfo->OptGrpConverter[i] = ucnv_loadSharedData(OptGroupByteToCPName[i], &stackPieces, &stackArgs, err); |
652 | 0 | } |
653 | 0 | } |
654 | |
|
655 | 0 | if(U_FAILURE(*err) || pArgs->onlyTestIsLoadable) { |
656 | 0 | _LMBCSClose(_this); |
657 | 0 | return; |
658 | 0 | } |
659 | 0 | extraInfo->OptGroup = OptGroup; |
660 | 0 | extraInfo->localeConverterIndex = FindLMBCSLocale(pArgs->locale); |
661 | 0 | } |
662 | 0 | else |
663 | 0 | { |
664 | 0 | *err = U_MEMORY_ALLOCATION_ERROR; |
665 | 0 | } |
666 | 0 | } |
667 | | |
668 | | U_CDECL_BEGIN |
669 | | static void U_CALLCONV |
670 | | _LMBCSClose(UConverter * _this) |
671 | 0 | { |
672 | 0 | if (_this->extraInfo != NULL) |
673 | 0 | { |
674 | 0 | ulmbcs_byte_t Ix; |
675 | 0 | UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) _this->extraInfo; |
676 | |
|
677 | 0 | for (Ix=0; Ix <= ULMBCS_GRP_LAST; Ix++) |
678 | 0 | { |
679 | 0 | if (extraInfo->OptGrpConverter[Ix] != NULL) |
680 | 0 | ucnv_unloadSharedDataIfReady(extraInfo->OptGrpConverter[Ix]); |
681 | 0 | } |
682 | 0 | if (!_this->isExtraLocal) { |
683 | 0 | uprv_free (_this->extraInfo); |
684 | 0 | _this->extraInfo = NULL; |
685 | 0 | } |
686 | 0 | } |
687 | 0 | } |
688 | | |
689 | | typedef struct LMBCSClone { |
690 | | UConverter cnv; |
691 | | UConverterDataLMBCS lmbcs; |
692 | | } LMBCSClone; |
693 | | |
694 | | static UConverter * U_CALLCONV |
695 | | _LMBCSSafeClone(const UConverter *cnv, |
696 | | void *stackBuffer, |
697 | | int32_t *pBufferSize, |
698 | 0 | UErrorCode *status) { |
699 | 0 | (void)status; |
700 | 0 | LMBCSClone *newLMBCS; |
701 | 0 | UConverterDataLMBCS *extraInfo; |
702 | 0 | int32_t i; |
703 | |
|
704 | 0 | if(*pBufferSize<=0) { |
705 | 0 | *pBufferSize=(int32_t)sizeof(LMBCSClone); |
706 | 0 | return NULL; |
707 | 0 | } |
708 | | |
709 | 0 | extraInfo=(UConverterDataLMBCS *)cnv->extraInfo; |
710 | 0 | newLMBCS=(LMBCSClone *)stackBuffer; |
711 | | |
712 | | /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ |
713 | |
|
714 | 0 | uprv_memcpy(&newLMBCS->lmbcs, extraInfo, sizeof(UConverterDataLMBCS)); |
715 | | |
716 | | /* share the subconverters */ |
717 | 0 | for(i = 0; i <= ULMBCS_GRP_LAST; ++i) { |
718 | 0 | if(extraInfo->OptGrpConverter[i] != NULL) { |
719 | 0 | ucnv_incrementRefCount(extraInfo->OptGrpConverter[i]); |
720 | 0 | } |
721 | 0 | } |
722 | |
|
723 | 0 | newLMBCS->cnv.extraInfo = &newLMBCS->lmbcs; |
724 | 0 | newLMBCS->cnv.isExtraLocal = TRUE; |
725 | 0 | return &newLMBCS->cnv; |
726 | 0 | } |
727 | | |
728 | | /* |
729 | | * There used to be a _LMBCSGetUnicodeSet() function here (up to svn revision 20117) |
730 | | * which added all code points except for U+F6xx |
731 | | * because those cannot be represented in the Unicode group. |
732 | | * However, it turns out that windows-950 has roundtrips for all of U+F6xx |
733 | | * which means that LMBCS can convert all Unicode code points after all. |
734 | | * We now simply use ucnv_getCompleteUnicodeSet(). |
735 | | * |
736 | | * This may need to be looked at again as Lotus uses _LMBCSGetUnicodeSet(). (091216) |
737 | | */ |
738 | | |
739 | | /* |
740 | | Here's the basic helper function that we use when converting from |
741 | | Unicode to LMBCS, and we suspect that a Unicode character will fit into |
742 | | one of the 12 groups. The return value is the number of bytes written |
743 | | starting at pStartLMBCS (if any). |
744 | | */ |
745 | | |
746 | | static size_t |
747 | | LMBCSConversionWorker ( |
748 | | UConverterDataLMBCS * extraInfo, /* subconverters, opt & locale groups */ |
749 | | ulmbcs_byte_t group, /* The group to try */ |
750 | | ulmbcs_byte_t * pStartLMBCS, /* where to put the results */ |
751 | | UChar * pUniChar, /* The input unicode character */ |
752 | | ulmbcs_byte_t * lastConverterIndex, /* output: track last successful group used */ |
753 | | UBool * groups_tried /* output: track any unsuccessful groups */ |
754 | | ) |
755 | 0 | { |
756 | 0 | ulmbcs_byte_t * pLMBCS = pStartLMBCS; |
757 | 0 | UConverterSharedData * xcnv = extraInfo->OptGrpConverter[group]; |
758 | |
|
759 | 0 | int bytesConverted; |
760 | 0 | uint32_t value; |
761 | 0 | ulmbcs_byte_t firstByte; |
762 | |
|
763 | 0 | U_ASSERT(xcnv); |
764 | 0 | U_ASSERT(group<ULMBCS_GRP_UNICODE); |
765 | |
|
766 | 0 | bytesConverted = ucnv_MBCSFromUChar32(xcnv, *pUniChar, &value, FALSE); |
767 | | |
768 | | /* get the first result byte */ |
769 | 0 | if(bytesConverted > 0) { |
770 | 0 | firstByte = (ulmbcs_byte_t)(value >> ((bytesConverted - 1) * 8)); |
771 | 0 | } else { |
772 | | /* most common failure mode is an unassigned character */ |
773 | 0 | groups_tried[group] = TRUE; |
774 | 0 | return 0; |
775 | 0 | } |
776 | | |
777 | 0 | *lastConverterIndex = group; |
778 | | |
779 | | /* All initial byte values in lower ascii range should have been caught by now, |
780 | | except with the exception group. |
781 | | */ |
782 | 0 | U_ASSERT((firstByte <= ULMBCS_C0END) || (firstByte >= ULMBCS_C1START) || (group == ULMBCS_GRP_EXCEPT)); |
783 | | |
784 | | /* use converted data: first write 0, 1 or two group bytes */ |
785 | 0 | if (group != ULMBCS_GRP_EXCEPT && extraInfo->OptGroup != group) |
786 | 0 | { |
787 | 0 | *pLMBCS++ = group; |
788 | 0 | if (bytesConverted == 1 && group >= ULMBCS_DOUBLEOPTGROUP_START) |
789 | 0 | { |
790 | 0 | *pLMBCS++ = group; |
791 | 0 | } |
792 | 0 | } |
793 | | |
794 | | /* don't emit control chars */ |
795 | 0 | if ( bytesConverted == 1 && firstByte < 0x20 ) |
796 | 0 | return 0; |
797 | | |
798 | | |
799 | | /* then move over the converted data */ |
800 | 0 | switch(bytesConverted) |
801 | 0 | { |
802 | 0 | case 4: |
803 | 0 | *pLMBCS++ = (ulmbcs_byte_t)(value >> 24); |
804 | 0 | U_FALLTHROUGH; |
805 | 0 | case 3: |
806 | 0 | *pLMBCS++ = (ulmbcs_byte_t)(value >> 16); |
807 | 0 | U_FALLTHROUGH; |
808 | 0 | case 2: |
809 | 0 | *pLMBCS++ = (ulmbcs_byte_t)(value >> 8); |
810 | 0 | U_FALLTHROUGH; |
811 | 0 | case 1: |
812 | 0 | *pLMBCS++ = (ulmbcs_byte_t)value; |
813 | 0 | U_FALLTHROUGH; |
814 | 0 | default: |
815 | | /* will never occur */ |
816 | 0 | break; |
817 | 0 | } |
818 | | |
819 | 0 | return (pLMBCS - pStartLMBCS); |
820 | 0 | } |
821 | | |
822 | | |
823 | | /* This is a much simpler version of above, when we |
824 | | know we are writing LMBCS using the Unicode group |
825 | | */ |
826 | | static size_t |
827 | | LMBCSConvertUni(ulmbcs_byte_t * pLMBCS, UChar uniChar) |
828 | 0 | { |
829 | | /* encode into LMBCS Unicode range */ |
830 | 0 | uint8_t LowCh = (uint8_t)(uniChar & 0x00FF); |
831 | 0 | uint8_t HighCh = (uint8_t)(uniChar >> 8); |
832 | |
|
833 | 0 | *pLMBCS++ = ULMBCS_GRP_UNICODE; |
834 | |
|
835 | 0 | if (LowCh == 0) |
836 | 0 | { |
837 | 0 | *pLMBCS++ = ULMBCS_UNICOMPATZERO; |
838 | 0 | *pLMBCS++ = HighCh; |
839 | 0 | } |
840 | 0 | else |
841 | 0 | { |
842 | 0 | *pLMBCS++ = HighCh; |
843 | 0 | *pLMBCS++ = LowCh; |
844 | 0 | } |
845 | 0 | return ULMBCS_UNICODE_SIZE; |
846 | 0 | } |
847 | | |
848 | | |
849 | | |
850 | | /* The main Unicode to LMBCS conversion function */ |
851 | | static void U_CALLCONV |
852 | | _LMBCSFromUnicode(UConverterFromUnicodeArgs* args, |
853 | | UErrorCode* err) |
854 | 0 | { |
855 | 0 | ulmbcs_byte_t lastConverterIndex = 0; |
856 | 0 | UChar uniChar; |
857 | 0 | ulmbcs_byte_t LMBCS[ULMBCS_CHARSIZE_MAX]; |
858 | 0 | ulmbcs_byte_t * pLMBCS; |
859 | 0 | int32_t bytes_written; |
860 | 0 | UBool groups_tried[ULMBCS_GRP_LAST+1]; |
861 | 0 | UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; |
862 | 0 | int sourceIndex = 0; |
863 | | |
864 | | /* Basic strategy: attempt to fill in local LMBCS 1-char buffer.(LMBCS) |
865 | | If that succeeds, see if it will all fit into the target & copy it over |
866 | | if it does. |
867 | | |
868 | | We try conversions in the following order: |
869 | | |
870 | | 1. Single-byte ascii & special fixed control chars (&null) |
871 | | 2. Look up group in table & try that (could be |
872 | | A) Unicode group |
873 | | B) control group, |
874 | | C) national encoding, |
875 | | or ambiguous SBCS or MBCS group (on to step 4...) |
876 | | |
877 | | 3. If its ambiguous, try this order: |
878 | | A) The optimization group |
879 | | B) The locale group |
880 | | C) The last group that succeeded with this string. |
881 | | D) every other group that's relevant (single or double) |
882 | | E) If its single-byte ambiguous, try the exceptions group |
883 | | |
884 | | 4. And as a grand fallback: Unicode |
885 | | */ |
886 | | |
887 | | /*Fix for SPR#DJOE66JFN3 (Lotus)*/ |
888 | 0 | ulmbcs_byte_t OldConverterIndex = 0; |
889 | |
|
890 | 0 | while (args->source < args->sourceLimit && !U_FAILURE(*err)) |
891 | 0 | { |
892 | | /*Fix for SPR#DJOE66JFN3 (Lotus)*/ |
893 | 0 | OldConverterIndex = extraInfo->localeConverterIndex; |
894 | |
|
895 | 0 | if (args->target >= args->targetLimit) |
896 | 0 | { |
897 | 0 | *err = U_BUFFER_OVERFLOW_ERROR; |
898 | 0 | break; |
899 | 0 | } |
900 | 0 | uniChar = *(args->source); |
901 | 0 | bytes_written = 0; |
902 | 0 | pLMBCS = LMBCS; |
903 | | |
904 | | /* check cases in rough order of how common they are, for speed */ |
905 | | |
906 | | /* single byte matches: strategy 1 */ |
907 | | /*Fix for SPR#DJOE66JFN3 (Lotus)*/ |
908 | 0 | if((uniChar>=0x80) && (uniChar<=0xff) |
909 | 0 | /*Fix for SPR#JUYA6XAERU and TSAO7GL5NK (Lotus)*/ &&(uniChar!=0xB1) &&(uniChar!=0xD7) &&(uniChar!=0xF7) |
910 | 0 | &&(uniChar!=0xB0) &&(uniChar!=0xB4) &&(uniChar!=0xB6) &&(uniChar!=0xA7) &&(uniChar!=0xA8)) |
911 | 0 | { |
912 | 0 | extraInfo->localeConverterIndex = ULMBCS_GRP_L1; |
913 | 0 | } |
914 | 0 | if (((uniChar > ULMBCS_C0END) && (uniChar < ULMBCS_C1START)) || |
915 | 0 | uniChar == 0 || uniChar == ULMBCS_HT || uniChar == ULMBCS_CR || |
916 | 0 | uniChar == ULMBCS_LF || uniChar == ULMBCS_123SYSTEMRANGE |
917 | 0 | ) |
918 | 0 | { |
919 | 0 | *pLMBCS++ = (ulmbcs_byte_t ) uniChar; |
920 | 0 | bytes_written = 1; |
921 | 0 | } |
922 | | |
923 | |
|
924 | 0 | if (!bytes_written) |
925 | 0 | { |
926 | | /* Check by UNICODE range (Strategy 2) */ |
927 | 0 | ulmbcs_byte_t group = FindLMBCSUniRange(uniChar); |
928 | | |
929 | 0 | if (group == ULMBCS_GRP_UNICODE) /* (Strategy 2A) */ |
930 | 0 | { |
931 | 0 | pLMBCS += LMBCSConvertUni(pLMBCS,uniChar); |
932 | | |
933 | 0 | bytes_written = (int32_t)(pLMBCS - LMBCS); |
934 | 0 | } |
935 | 0 | else if (group == ULMBCS_GRP_CTRL) /* (Strategy 2B) */ |
936 | 0 | { |
937 | | /* Handle control characters here */ |
938 | 0 | if (uniChar <= ULMBCS_C0END) |
939 | 0 | { |
940 | 0 | *pLMBCS++ = ULMBCS_GRP_CTRL; |
941 | 0 | *pLMBCS++ = (ulmbcs_byte_t)(ULMBCS_CTRLOFFSET + uniChar); |
942 | 0 | } |
943 | 0 | else if (uniChar >= ULMBCS_C1START && uniChar <= ULMBCS_C1START + ULMBCS_CTRLOFFSET) |
944 | 0 | { |
945 | 0 | *pLMBCS++ = ULMBCS_GRP_CTRL; |
946 | 0 | *pLMBCS++ = (ulmbcs_byte_t ) (uniChar & 0x00FF); |
947 | 0 | } |
948 | 0 | bytes_written = (int32_t)(pLMBCS - LMBCS); |
949 | 0 | } |
950 | 0 | else if (group < ULMBCS_GRP_UNICODE) /* (Strategy 2C) */ |
951 | 0 | { |
952 | | /* a specific converter has been identified - use it */ |
953 | 0 | bytes_written = (int32_t)LMBCSConversionWorker ( |
954 | 0 | extraInfo, group, pLMBCS, &uniChar, |
955 | 0 | &lastConverterIndex, groups_tried); |
956 | 0 | } |
957 | 0 | if (!bytes_written) /* the ambiguous group cases (Strategy 3) */ |
958 | 0 | { |
959 | 0 | uprv_memset(groups_tried, 0, sizeof(groups_tried)); |
960 | | |
961 | | /* check for non-default optimization group (Strategy 3A )*/ |
962 | 0 | if ((extraInfo->OptGroup != 1) && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->OptGroup))) |
963 | 0 | { |
964 | | /*zhujin: upgrade, merge #39299 here (Lotus) */ |
965 | | /*To make R5 compatible translation, look for exceptional group first for non-DBCS*/ |
966 | |
|
967 | 0 | if(extraInfo->localeConverterIndex < ULMBCS_DOUBLEOPTGROUP_START) |
968 | 0 | { |
969 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
970 | 0 | ULMBCS_GRP_L1, pLMBCS, &uniChar, |
971 | 0 | &lastConverterIndex, groups_tried); |
972 | |
|
973 | 0 | if(!bytes_written) |
974 | 0 | { |
975 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
976 | 0 | ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar, |
977 | 0 | &lastConverterIndex, groups_tried); |
978 | 0 | } |
979 | 0 | if(!bytes_written) |
980 | 0 | { |
981 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
982 | 0 | extraInfo->localeConverterIndex, pLMBCS, &uniChar, |
983 | 0 | &lastConverterIndex, groups_tried); |
984 | 0 | } |
985 | 0 | } |
986 | 0 | else |
987 | 0 | { |
988 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
989 | 0 | extraInfo->localeConverterIndex, pLMBCS, &uniChar, |
990 | 0 | &lastConverterIndex, groups_tried); |
991 | 0 | } |
992 | 0 | } |
993 | | /* check for locale optimization group (Strategy 3B) */ |
994 | 0 | if (!bytes_written && (extraInfo->localeConverterIndex) && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->localeConverterIndex))) |
995 | 0 | { |
996 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
997 | 0 | extraInfo->localeConverterIndex, pLMBCS, &uniChar, &lastConverterIndex, groups_tried); |
998 | 0 | } |
999 | | /* check for last optimization group used for this string (Strategy 3C) */ |
1000 | 0 | if (!bytes_written && (lastConverterIndex) && (ULMBCS_AMBIGUOUS_MATCH(group, lastConverterIndex))) |
1001 | 0 | { |
1002 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
1003 | 0 | lastConverterIndex, pLMBCS, &uniChar, &lastConverterIndex, groups_tried); |
1004 | 0 | } |
1005 | 0 | if (!bytes_written) |
1006 | 0 | { |
1007 | | /* just check every possible matching converter (Strategy 3D) */ |
1008 | 0 | ulmbcs_byte_t grp_start; |
1009 | 0 | ulmbcs_byte_t grp_end; |
1010 | 0 | ulmbcs_byte_t grp_ix; |
1011 | 0 | grp_start = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS) |
1012 | 0 | ? ULMBCS_DOUBLEOPTGROUP_START |
1013 | 0 | : ULMBCS_GRP_L1); |
1014 | 0 | grp_end = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS) |
1015 | 0 | ? ULMBCS_GRP_LAST |
1016 | 0 | : ULMBCS_GRP_TH); |
1017 | 0 | if(group == ULMBCS_AMBIGUOUS_ALL) |
1018 | 0 | { |
1019 | 0 | grp_start = ULMBCS_GRP_L1; |
1020 | 0 | grp_end = ULMBCS_GRP_LAST; |
1021 | 0 | } |
1022 | 0 | for (grp_ix = grp_start; |
1023 | 0 | grp_ix <= grp_end && !bytes_written; |
1024 | 0 | grp_ix++) |
1025 | 0 | { |
1026 | 0 | if (extraInfo->OptGrpConverter [grp_ix] && !groups_tried [grp_ix]) |
1027 | 0 | { |
1028 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
1029 | 0 | grp_ix, pLMBCS, &uniChar, |
1030 | 0 | &lastConverterIndex, groups_tried); |
1031 | 0 | } |
1032 | 0 | } |
1033 | | /* a final conversion fallback to the exceptions group if its likely |
1034 | | to be single byte (Strategy 3E) */ |
1035 | 0 | if (!bytes_written && grp_start == ULMBCS_GRP_L1) |
1036 | 0 | { |
1037 | 0 | bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, |
1038 | 0 | ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar, |
1039 | 0 | &lastConverterIndex, groups_tried); |
1040 | 0 | } |
1041 | 0 | } |
1042 | | /* all of our other strategies failed. Fallback to Unicode. (Strategy 4)*/ |
1043 | 0 | if (!bytes_written) |
1044 | 0 | { |
1045 | |
|
1046 | 0 | pLMBCS += LMBCSConvertUni(pLMBCS, uniChar); |
1047 | 0 | bytes_written = (int32_t)(pLMBCS - LMBCS); |
1048 | 0 | } |
1049 | 0 | } |
1050 | 0 | } |
1051 | | |
1052 | | /* we have a translation. increment source and write as much as possible to target */ |
1053 | 0 | args->source++; |
1054 | 0 | pLMBCS = LMBCS; |
1055 | 0 | while (args->target < args->targetLimit && bytes_written--) |
1056 | 0 | { |
1057 | 0 | *(args->target)++ = *pLMBCS++; |
1058 | 0 | if (args->offsets) |
1059 | 0 | { |
1060 | 0 | *(args->offsets)++ = sourceIndex; |
1061 | 0 | } |
1062 | 0 | } |
1063 | 0 | sourceIndex++; |
1064 | 0 | if (bytes_written > 0) |
1065 | 0 | { |
1066 | | /* write any bytes that didn't fit in target to the error buffer, |
1067 | | common code will move this to target if we get called back with |
1068 | | enough target room |
1069 | | */ |
1070 | 0 | uint8_t * pErrorBuffer = args->converter->charErrorBuffer; |
1071 | 0 | *err = U_BUFFER_OVERFLOW_ERROR; |
1072 | 0 | args->converter->charErrorBufferLength = (int8_t)bytes_written; |
1073 | 0 | while (bytes_written--) |
1074 | 0 | { |
1075 | 0 | *pErrorBuffer++ = *pLMBCS++; |
1076 | 0 | } |
1077 | 0 | } |
1078 | | /*Fix for SPR#DJOE66JFN3 (Lotus)*/ |
1079 | 0 | extraInfo->localeConverterIndex = OldConverterIndex; |
1080 | 0 | } |
1081 | 0 | } |
1082 | | |
1083 | | |
1084 | | /* Now, the Unicode from LMBCS section */ |
1085 | | |
1086 | | |
1087 | | /* A function to call when we are looking at the Unicode group byte in LMBCS */ |
1088 | | static UChar |
1089 | | GetUniFromLMBCSUni(char const ** ppLMBCSin) /* Called with LMBCS-style Unicode byte stream */ |
1090 | 0 | { |
1091 | 0 | uint8_t HighCh = *(*ppLMBCSin)++; /* Big-endian Unicode in LMBCS compatibility group*/ |
1092 | 0 | uint8_t LowCh = *(*ppLMBCSin)++; |
1093 | |
|
1094 | 0 | if (HighCh == ULMBCS_UNICOMPATZERO ) |
1095 | 0 | { |
1096 | 0 | HighCh = LowCh; |
1097 | 0 | LowCh = 0; /* zero-byte in LSB special character */ |
1098 | 0 | } |
1099 | 0 | return (UChar)((HighCh << 8) | LowCh); |
1100 | 0 | } |
1101 | | |
1102 | | |
1103 | | |
1104 | | /* CHECK_SOURCE_LIMIT: Helper macro to verify that there are at least'index' |
1105 | | bytes left in source up to sourceLimit.Errors appropriately if not. |
1106 | | If we reach the limit, then update the source pointer to there to consume |
1107 | | all input as required by ICU converter semantics. |
1108 | | */ |
1109 | | |
1110 | 0 | #define CHECK_SOURCE_LIMIT(index) UPRV_BLOCK_MACRO_BEGIN { \ |
1111 | 0 | if (args->source+index > args->sourceLimit) { \ |
1112 | 0 | *err = U_TRUNCATED_CHAR_FOUND; \ |
1113 | 0 | args->source = args->sourceLimit; \ |
1114 | 0 | return 0xffff; \ |
1115 | 0 | } \ |
1116 | 0 | } UPRV_BLOCK_MACRO_END |
1117 | | |
1118 | | /* Return the Unicode representation for the current LMBCS character */ |
1119 | | |
1120 | | static UChar32 U_CALLCONV |
1121 | | _LMBCSGetNextUCharWorker(UConverterToUnicodeArgs* args, |
1122 | | UErrorCode* err) |
1123 | 0 | { |
1124 | 0 | UChar32 uniChar = 0; /* an output UNICODE char */ |
1125 | 0 | ulmbcs_byte_t CurByte; /* A byte from the input stream */ |
1126 | | |
1127 | | /* error check */ |
1128 | 0 | if (args->source >= args->sourceLimit) |
1129 | 0 | { |
1130 | 0 | *err = U_ILLEGAL_ARGUMENT_ERROR; |
1131 | 0 | return 0xffff; |
1132 | 0 | } |
1133 | | /* Grab first byte & save address for error recovery */ |
1134 | 0 | CurByte = *((ulmbcs_byte_t *) (args->source++)); |
1135 | | |
1136 | | /* |
1137 | | * at entry of each if clause: |
1138 | | * 1. 'CurByte' points at the first byte of a LMBCS character |
1139 | | * 2. '*source'points to the next byte of the source stream after 'CurByte' |
1140 | | * |
1141 | | * the job of each if clause is: |
1142 | | * 1. set '*source' to point at the beginning of next char (nop if LMBCS char is only 1 byte) |
1143 | | * 2. set 'uniChar' up with the right Unicode value, or set 'err' appropriately |
1144 | | */ |
1145 | | |
1146 | | /* First lets check the simple fixed values. */ |
1147 | |
|
1148 | 0 | if(((CurByte > ULMBCS_C0END) && (CurByte < ULMBCS_C1START)) /* ascii range */ |
1149 | 0 | || (CurByte == 0) |
1150 | 0 | || CurByte == ULMBCS_HT || CurByte == ULMBCS_CR |
1151 | 0 | || CurByte == ULMBCS_LF || CurByte == ULMBCS_123SYSTEMRANGE) |
1152 | 0 | { |
1153 | 0 | uniChar = CurByte; |
1154 | 0 | } |
1155 | 0 | else |
1156 | 0 | { |
1157 | 0 | UConverterDataLMBCS * extraInfo; |
1158 | 0 | ulmbcs_byte_t group; |
1159 | 0 | UConverterSharedData *cnv; |
1160 | | |
1161 | 0 | if (CurByte == ULMBCS_GRP_CTRL) /* Control character group - no opt group update */ |
1162 | 0 | { |
1163 | 0 | ulmbcs_byte_t C0C1byte; |
1164 | 0 | CHECK_SOURCE_LIMIT(1); |
1165 | 0 | C0C1byte = *(args->source)++; |
1166 | 0 | uniChar = (C0C1byte < ULMBCS_C1START) ? C0C1byte - ULMBCS_CTRLOFFSET : C0C1byte; |
1167 | 0 | } |
1168 | 0 | else |
1169 | 0 | if (CurByte == ULMBCS_GRP_UNICODE) /* Unicode compatibility group: BigEndian UTF16 */ |
1170 | 0 | { |
1171 | 0 | CHECK_SOURCE_LIMIT(2); |
1172 | | |
1173 | | /* don't check for error indicators fffe/ffff below */ |
1174 | 0 | return GetUniFromLMBCSUni(&(args->source)); |
1175 | 0 | } |
1176 | 0 | else if (CurByte <= ULMBCS_CTRLOFFSET) |
1177 | 0 | { |
1178 | 0 | group = CurByte; /* group byte is in the source */ |
1179 | 0 | extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; |
1180 | 0 | if (group > ULMBCS_GRP_LAST || (cnv = extraInfo->OptGrpConverter[group]) == NULL) |
1181 | 0 | { |
1182 | | /* this is not a valid group byte - no converter*/ |
1183 | 0 | *err = U_INVALID_CHAR_FOUND; |
1184 | 0 | } |
1185 | 0 | else if (group >= ULMBCS_DOUBLEOPTGROUP_START) /* double byte conversion */ |
1186 | 0 | { |
1187 | |
|
1188 | 0 | CHECK_SOURCE_LIMIT(2); |
1189 | | |
1190 | | /* check for LMBCS doubled-group-byte case */ |
1191 | 0 | if (*args->source == group) { |
1192 | | /* single byte */ |
1193 | 0 | ++args->source; |
1194 | 0 | uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 1, FALSE); |
1195 | 0 | ++args->source; |
1196 | 0 | } else { |
1197 | | /* double byte */ |
1198 | 0 | uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 2, FALSE); |
1199 | 0 | args->source += 2; |
1200 | 0 | } |
1201 | 0 | } |
1202 | 0 | else { /* single byte conversion */ |
1203 | 0 | CHECK_SOURCE_LIMIT(1); |
1204 | 0 | CurByte = *(args->source)++; |
1205 | | |
1206 | 0 | if (CurByte >= ULMBCS_C1START) |
1207 | 0 | { |
1208 | 0 | uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte); |
1209 | 0 | } |
1210 | 0 | else |
1211 | 0 | { |
1212 | | /* The non-optimizable oddballs where there is an explicit byte |
1213 | | * AND the second byte is not in the upper ascii range |
1214 | | */ |
1215 | 0 | char bytes[2]; |
1216 | |
|
1217 | 0 | extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; |
1218 | 0 | cnv = extraInfo->OptGrpConverter [ULMBCS_GRP_EXCEPT]; |
1219 | | |
1220 | | /* Lookup value must include opt group */ |
1221 | 0 | bytes[0] = group; |
1222 | 0 | bytes[1] = CurByte; |
1223 | 0 | uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, bytes, 2, FALSE); |
1224 | 0 | } |
1225 | 0 | } |
1226 | 0 | } |
1227 | 0 | else if (CurByte >= ULMBCS_C1START) /* group byte is implicit */ |
1228 | 0 | { |
1229 | 0 | extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; |
1230 | 0 | group = extraInfo->OptGroup; |
1231 | 0 | cnv = extraInfo->OptGrpConverter[group]; |
1232 | 0 | if (group >= ULMBCS_DOUBLEOPTGROUP_START) /* double byte conversion */ |
1233 | 0 | { |
1234 | 0 | if (!ucnv_MBCSIsLeadByte(cnv, CurByte)) |
1235 | 0 | { |
1236 | 0 | CHECK_SOURCE_LIMIT(0); |
1237 | | |
1238 | | /* let the MBCS conversion consume CurByte again */ |
1239 | 0 | uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 1, FALSE); |
1240 | 0 | } |
1241 | 0 | else |
1242 | 0 | { |
1243 | 0 | CHECK_SOURCE_LIMIT(1); |
1244 | | /* let the MBCS conversion consume CurByte again */ |
1245 | 0 | uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 2, FALSE); |
1246 | 0 | ++args->source; |
1247 | 0 | } |
1248 | 0 | } |
1249 | 0 | else /* single byte conversion */ |
1250 | 0 | { |
1251 | 0 | uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte); |
1252 | 0 | } |
1253 | 0 | } |
1254 | 0 | } |
1255 | 0 | return uniChar; |
1256 | 0 | } |
1257 | | |
1258 | | |
1259 | | /* The exported function that converts lmbcs to one or more |
1260 | | UChars - currently UTF-16 |
1261 | | */ |
1262 | | static void U_CALLCONV |
1263 | | _LMBCSToUnicodeWithOffsets(UConverterToUnicodeArgs* args, |
1264 | | UErrorCode* err) |
1265 | 0 | { |
1266 | 0 | char LMBCS [ULMBCS_CHARSIZE_MAX]; |
1267 | 0 | UChar uniChar; /* one output UNICODE char */ |
1268 | 0 | const char * saveSource; /* beginning of current code point */ |
1269 | 0 | const char * pStartLMBCS = args->source; /* beginning of whole string */ |
1270 | 0 | const char * errSource = NULL; /* pointer to actual input in case an error occurs */ |
1271 | 0 | int8_t savebytes = 0; |
1272 | | |
1273 | | /* Process from source to limit, or until error */ |
1274 | 0 | while (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit > args->target) |
1275 | 0 | { |
1276 | 0 | saveSource = args->source; /* beginning of current code point */ |
1277 | |
|
1278 | 0 | if (args->converter->toULength) /* reassemble char from previous call */ |
1279 | 0 | { |
1280 | 0 | const char *saveSourceLimit; |
1281 | 0 | size_t size_old = args->converter->toULength; |
1282 | | |
1283 | | /* limit from source is either remainder of temp buffer, or user limit on source */ |
1284 | 0 | size_t size_new_maybe_1 = sizeof(LMBCS) - size_old; |
1285 | 0 | size_t size_new_maybe_2 = args->sourceLimit - args->source; |
1286 | 0 | size_t size_new = (size_new_maybe_1 < size_new_maybe_2) ? size_new_maybe_1 : size_new_maybe_2; |
1287 | | |
1288 | | |
1289 | 0 | uprv_memcpy(LMBCS, args->converter->toUBytes, size_old); |
1290 | 0 | uprv_memcpy(LMBCS + size_old, args->source, size_new); |
1291 | 0 | saveSourceLimit = args->sourceLimit; |
1292 | 0 | args->source = errSource = LMBCS; |
1293 | 0 | args->sourceLimit = LMBCS+size_old+size_new; |
1294 | 0 | savebytes = (int8_t)(size_old+size_new); |
1295 | 0 | uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err); |
1296 | 0 | args->source = saveSource + ((args->source - LMBCS) - size_old); |
1297 | 0 | args->sourceLimit = saveSourceLimit; |
1298 | |
|
1299 | 0 | if (*err == U_TRUNCATED_CHAR_FOUND) |
1300 | 0 | { |
1301 | | /* evil special case: source buffers so small a char spans more than 2 buffers */ |
1302 | 0 | args->converter->toULength = savebytes; |
1303 | 0 | uprv_memcpy(args->converter->toUBytes, LMBCS, savebytes); |
1304 | 0 | args->source = args->sourceLimit; |
1305 | 0 | *err = U_ZERO_ERROR; |
1306 | 0 | return; |
1307 | 0 | } |
1308 | 0 | else |
1309 | 0 | { |
1310 | | /* clear the partial-char marker */ |
1311 | 0 | args->converter->toULength = 0; |
1312 | 0 | } |
1313 | 0 | } |
1314 | 0 | else |
1315 | 0 | { |
1316 | 0 | errSource = saveSource; |
1317 | 0 | uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err); |
1318 | 0 | savebytes = (int8_t)(args->source - saveSource); |
1319 | 0 | } |
1320 | 0 | if (U_SUCCESS(*err)) |
1321 | 0 | { |
1322 | 0 | if (uniChar < 0xfffe) |
1323 | 0 | { |
1324 | 0 | *(args->target)++ = uniChar; |
1325 | 0 | if(args->offsets) |
1326 | 0 | { |
1327 | 0 | *(args->offsets)++ = (int32_t)(saveSource - pStartLMBCS); |
1328 | 0 | } |
1329 | 0 | } |
1330 | 0 | else if (uniChar == 0xfffe) |
1331 | 0 | { |
1332 | 0 | *err = U_INVALID_CHAR_FOUND; |
1333 | 0 | } |
1334 | 0 | else /* if (uniChar == 0xffff) */ |
1335 | 0 | { |
1336 | 0 | *err = U_ILLEGAL_CHAR_FOUND; |
1337 | 0 | } |
1338 | 0 | } |
1339 | 0 | } |
1340 | | /* if target ran out before source, return U_BUFFER_OVERFLOW_ERROR */ |
1341 | 0 | if (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit <= args->target) |
1342 | 0 | { |
1343 | 0 | *err = U_BUFFER_OVERFLOW_ERROR; |
1344 | 0 | } |
1345 | 0 | else if (U_FAILURE(*err)) |
1346 | 0 | { |
1347 | | /* If character incomplete or unmappable/illegal, store it in toUBytes[] */ |
1348 | 0 | args->converter->toULength = savebytes; |
1349 | 0 | if (savebytes > 0) { |
1350 | 0 | uprv_memcpy(args->converter->toUBytes, errSource, savebytes); |
1351 | 0 | } |
1352 | 0 | if (*err == U_TRUNCATED_CHAR_FOUND) { |
1353 | 0 | *err = U_ZERO_ERROR; |
1354 | 0 | } |
1355 | 0 | } |
1356 | 0 | } |
1357 | | |
1358 | | /* And now, the macroized declarations of data & functions: */ |
1359 | | DEFINE_LMBCS_OPEN(1) |
1360 | | DEFINE_LMBCS_OPEN(2) |
1361 | | DEFINE_LMBCS_OPEN(3) |
1362 | | DEFINE_LMBCS_OPEN(4) |
1363 | | DEFINE_LMBCS_OPEN(5) |
1364 | | DEFINE_LMBCS_OPEN(6) |
1365 | | DEFINE_LMBCS_OPEN(8) |
1366 | | DEFINE_LMBCS_OPEN(11) |
1367 | | DEFINE_LMBCS_OPEN(16) |
1368 | | DEFINE_LMBCS_OPEN(17) |
1369 | | DEFINE_LMBCS_OPEN(18) |
1370 | | DEFINE_LMBCS_OPEN(19) |
1371 | | |
1372 | | |
1373 | | DECLARE_LMBCS_DATA(1) |
1374 | | DECLARE_LMBCS_DATA(2) |
1375 | | DECLARE_LMBCS_DATA(3) |
1376 | | DECLARE_LMBCS_DATA(4) |
1377 | | DECLARE_LMBCS_DATA(5) |
1378 | | DECLARE_LMBCS_DATA(6) |
1379 | | DECLARE_LMBCS_DATA(8) |
1380 | | DECLARE_LMBCS_DATA(11) |
1381 | | DECLARE_LMBCS_DATA(16) |
1382 | | DECLARE_LMBCS_DATA(17) |
1383 | | DECLARE_LMBCS_DATA(18) |
1384 | | DECLARE_LMBCS_DATA(19) |
1385 | | |
1386 | | U_CDECL_END |
1387 | | |
1388 | | #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ |