/src/icu/icu4c/source/common/unistr_cnv.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // © 2016 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | /* |
4 | | ******************************************************************************* |
5 | | * |
6 | | * Copyright (C) 1999-2014, International Business Machines |
7 | | * Corporation and others. All Rights Reserved. |
8 | | * |
9 | | ******************************************************************************* |
10 | | * file name: unistr_cnv.cpp |
11 | | * encoding: UTF-8 |
12 | | * tab size: 8 (not used) |
13 | | * indentation:2 |
14 | | * |
15 | | * created on: 2004aug19 |
16 | | * created by: Markus W. Scherer |
17 | | * |
18 | | * Character conversion functions moved here from unistr.cpp |
19 | | */ |
20 | | |
21 | | #include "unicode/utypes.h" |
22 | | |
23 | | #if !UCONFIG_NO_CONVERSION |
24 | | |
25 | | #include "unicode/putil.h" |
26 | | #include "cstring.h" |
27 | | #include "cmemory.h" |
28 | | #include "unicode/ustring.h" |
29 | | #include "unicode/unistr.h" |
30 | | #include "unicode/ucnv.h" |
31 | | #include "ucnv_imp.h" |
32 | | #include "putilimp.h" |
33 | | #include "ustr_cnv.h" |
34 | | #include "ustr_imp.h" |
35 | | |
36 | | U_NAMESPACE_BEGIN |
37 | | |
38 | | //======================================== |
39 | | // Constructors |
40 | | //======================================== |
41 | | |
42 | | #if !U_CHARSET_IS_UTF8 |
43 | | |
44 | | UnicodeString::UnicodeString(const char *codepageData) { |
45 | | fUnion.fFields.fLengthAndFlags = kShortString; |
46 | | if(codepageData != 0) { |
47 | | doCodepageCreate(codepageData, (int32_t)uprv_strlen(codepageData), 0); |
48 | | } |
49 | | } |
50 | | |
51 | | UnicodeString::UnicodeString(const char *codepageData, |
52 | | int32_t dataLength) { |
53 | | fUnion.fFields.fLengthAndFlags = kShortString; |
54 | | if(codepageData != 0) { |
55 | | doCodepageCreate(codepageData, dataLength, 0); |
56 | | } |
57 | | } |
58 | | |
59 | | // else see unistr.cpp |
60 | | #endif |
61 | | |
62 | | UnicodeString::UnicodeString(const char *codepageData, |
63 | 0 | const char *codepage) { |
64 | 0 | fUnion.fFields.fLengthAndFlags = kShortString; |
65 | 0 | if(codepageData != 0) { |
66 | 0 | doCodepageCreate(codepageData, (int32_t)uprv_strlen(codepageData), codepage); |
67 | 0 | } |
68 | 0 | } |
69 | | |
70 | | UnicodeString::UnicodeString(const char *codepageData, |
71 | | int32_t dataLength, |
72 | 2.54k | const char *codepage) { |
73 | 2.54k | fUnion.fFields.fLengthAndFlags = kShortString; |
74 | 2.54k | if(codepageData != 0) { |
75 | 2.54k | doCodepageCreate(codepageData, dataLength, codepage); |
76 | 2.54k | } |
77 | 2.54k | } |
78 | | |
79 | | UnicodeString::UnicodeString(const char *src, int32_t srcLength, |
80 | | UConverter *cnv, |
81 | 0 | UErrorCode &errorCode) { |
82 | 0 | fUnion.fFields.fLengthAndFlags = kShortString; |
83 | 0 | if(U_SUCCESS(errorCode)) { |
84 | | // check arguments |
85 | 0 | if(src==nullptr) { |
86 | | // treat as an empty string, do nothing more |
87 | 0 | } else if(srcLength<-1) { |
88 | 0 | errorCode=U_ILLEGAL_ARGUMENT_ERROR; |
89 | 0 | } else { |
90 | | // get input length |
91 | 0 | if(srcLength==-1) { |
92 | 0 | srcLength=(int32_t)uprv_strlen(src); |
93 | 0 | } |
94 | 0 | if(srcLength>0) { |
95 | 0 | if(cnv!=0) { |
96 | | // use the provided converter |
97 | 0 | ucnv_resetToUnicode(cnv); |
98 | 0 | doCodepageCreate(src, srcLength, cnv, errorCode); |
99 | 0 | } else { |
100 | | // use the default converter |
101 | 0 | cnv=u_getDefaultConverter(&errorCode); |
102 | 0 | doCodepageCreate(src, srcLength, cnv, errorCode); |
103 | 0 | u_releaseDefaultConverter(cnv); |
104 | 0 | } |
105 | 0 | } |
106 | 0 | } |
107 | |
|
108 | 0 | if(U_FAILURE(errorCode)) { |
109 | 0 | setToBogus(); |
110 | 0 | } |
111 | 0 | } |
112 | 0 | } |
113 | | |
114 | | //======================================== |
115 | | // Codeset conversion |
116 | | //======================================== |
117 | | |
118 | | #if !U_CHARSET_IS_UTF8 |
119 | | |
120 | | int32_t |
121 | | UnicodeString::extract(int32_t start, |
122 | | int32_t length, |
123 | | char *target, |
124 | | uint32_t dstSize) const { |
125 | | return extract(start, length, target, dstSize, 0); |
126 | | } |
127 | | |
128 | | // else see unistr.cpp |
129 | | #endif |
130 | | |
131 | | int32_t |
132 | | UnicodeString::extract(int32_t start, |
133 | | int32_t length, |
134 | | char *target, |
135 | | uint32_t dstSize, |
136 | | const char *codepage) const |
137 | 0 | { |
138 | | // if the arguments are illegal, then do nothing |
139 | 0 | if(/*dstSize < 0 || */(dstSize > 0 && target == 0)) { |
140 | 0 | return 0; |
141 | 0 | } |
142 | | |
143 | | // pin the indices to legal values |
144 | 0 | pinIndices(start, length); |
145 | | |
146 | | // We need to cast dstSize to int32_t for all subsequent code. |
147 | | // I don't know why the API was defined with uint32_t but we are stuck with it. |
148 | | // Also, dstSize==0xffffffff means "unlimited" but if we use target+dstSize |
149 | | // as a limit in some functions, it may wrap around and yield a pointer |
150 | | // that compares less-than target. |
151 | 0 | int32_t capacity; |
152 | 0 | if(dstSize < 0x7fffffff) { |
153 | | // Assume that the capacity is real and a limit pointer won't wrap around. |
154 | 0 | capacity = (int32_t)dstSize; |
155 | 0 | } else { |
156 | | // Pin the capacity so that a limit pointer does not wrap around. |
157 | 0 | char *targetLimit = (char *)U_MAX_PTR(target); |
158 | | // U_MAX_PTR(target) returns a targetLimit that is at most 0x7fffffff |
159 | | // greater than target and does not wrap around the top of the address space. |
160 | 0 | capacity = (int32_t)(targetLimit - target); |
161 | 0 | } |
162 | | |
163 | | // create the converter |
164 | 0 | UConverter *converter; |
165 | 0 | UErrorCode status = U_ZERO_ERROR; |
166 | | |
167 | | // just write the NUL if the string length is 0 |
168 | 0 | if(length == 0) { |
169 | 0 | return u_terminateChars(target, capacity, 0, &status); |
170 | 0 | } |
171 | | |
172 | | // if the codepage is the default, use our cache |
173 | | // if it is an empty string, then use the "invariant character" conversion |
174 | 0 | if (codepage == 0) { |
175 | 0 | const char *defaultName = ucnv_getDefaultName(); |
176 | 0 | if(UCNV_FAST_IS_UTF8(defaultName)) { |
177 | 0 | return toUTF8(start, length, target, capacity); |
178 | 0 | } |
179 | 0 | converter = u_getDefaultConverter(&status); |
180 | 0 | } else if (*codepage == 0) { |
181 | | // use the "invariant characters" conversion |
182 | 0 | int32_t destLength; |
183 | 0 | if(length <= capacity) { |
184 | 0 | destLength = length; |
185 | 0 | } else { |
186 | 0 | destLength = capacity; |
187 | 0 | } |
188 | 0 | u_UCharsToChars(getArrayStart() + start, target, destLength); |
189 | 0 | return u_terminateChars(target, capacity, length, &status); |
190 | 0 | } else { |
191 | 0 | converter = ucnv_open(codepage, &status); |
192 | 0 | } |
193 | | |
194 | 0 | length = doExtract(start, length, target, capacity, converter, status); |
195 | | |
196 | | // close the converter |
197 | 0 | if (codepage == 0) { |
198 | 0 | u_releaseDefaultConverter(converter); |
199 | 0 | } else { |
200 | 0 | ucnv_close(converter); |
201 | 0 | } |
202 | |
|
203 | 0 | return length; |
204 | 0 | } |
205 | | |
206 | | int32_t |
207 | | UnicodeString::extract(char *dest, int32_t destCapacity, |
208 | | UConverter *cnv, |
209 | | UErrorCode &errorCode) const |
210 | 8.03k | { |
211 | 8.03k | if(U_FAILURE(errorCode)) { |
212 | 0 | return 0; |
213 | 0 | } |
214 | | |
215 | 8.03k | if(isBogus() || destCapacity<0 || (destCapacity>0 && dest==0)) { |
216 | 0 | errorCode=U_ILLEGAL_ARGUMENT_ERROR; |
217 | 0 | return 0; |
218 | 0 | } |
219 | | |
220 | | // nothing to do? |
221 | 8.03k | if(isEmpty()) { |
222 | 72 | return u_terminateChars(dest, destCapacity, 0, &errorCode); |
223 | 72 | } |
224 | | |
225 | | // get the converter |
226 | 7.96k | UBool isDefaultConverter; |
227 | 7.96k | if(cnv==0) { |
228 | 0 | isDefaultConverter=true; |
229 | 0 | cnv=u_getDefaultConverter(&errorCode); |
230 | 0 | if(U_FAILURE(errorCode)) { |
231 | 0 | return 0; |
232 | 0 | } |
233 | 7.96k | } else { |
234 | 7.96k | isDefaultConverter=false; |
235 | 7.96k | ucnv_resetFromUnicode(cnv); |
236 | 7.96k | } |
237 | | |
238 | | // convert |
239 | 7.96k | int32_t len=doExtract(0, length(), dest, destCapacity, cnv, errorCode); |
240 | | |
241 | | // release the converter |
242 | 7.96k | if(isDefaultConverter) { |
243 | 0 | u_releaseDefaultConverter(cnv); |
244 | 0 | } |
245 | | |
246 | 7.96k | return len; |
247 | 7.96k | } |
248 | | |
249 | | int32_t |
250 | | UnicodeString::doExtract(int32_t start, int32_t length, |
251 | | char *dest, int32_t destCapacity, |
252 | | UConverter *cnv, |
253 | | UErrorCode &errorCode) const |
254 | 7.96k | { |
255 | 7.96k | if(U_FAILURE(errorCode)) { |
256 | 0 | if(destCapacity!=0) { |
257 | 0 | *dest=0; |
258 | 0 | } |
259 | 0 | return 0; |
260 | 0 | } |
261 | | |
262 | 7.96k | const char16_t *src=getArrayStart()+start, *srcLimit=src+length; |
263 | 7.96k | char *originalDest=dest; |
264 | 7.96k | const char *destLimit; |
265 | | |
266 | 7.96k | if(destCapacity==0) { |
267 | 0 | destLimit=dest=0; |
268 | 7.96k | } else if(destCapacity==-1) { |
269 | | // Pin the limit to U_MAX_PTR if the "magic" destCapacity is used. |
270 | 0 | destLimit=(char*)U_MAX_PTR(dest); |
271 | | // for NUL-termination, translate into highest int32_t |
272 | 0 | destCapacity=0x7fffffff; |
273 | 7.96k | } else { |
274 | 7.96k | destLimit=dest+destCapacity; |
275 | 7.96k | } |
276 | | |
277 | | // perform the conversion |
278 | 7.96k | ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, true, &errorCode); |
279 | 7.96k | length=(int32_t)(dest-originalDest); |
280 | | |
281 | | // if an overflow occurs, then get the preflighting length |
282 | 7.96k | if(errorCode==U_BUFFER_OVERFLOW_ERROR) { |
283 | 904 | char buffer[1024]; |
284 | | |
285 | 904 | destLimit=buffer+sizeof(buffer); |
286 | 31.4k | do { |
287 | 31.4k | dest=buffer; |
288 | 31.4k | errorCode=U_ZERO_ERROR; |
289 | 31.4k | ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, true, &errorCode); |
290 | 31.4k | length+=(int32_t)(dest-buffer); |
291 | 31.4k | } while(errorCode==U_BUFFER_OVERFLOW_ERROR); |
292 | 904 | } |
293 | | |
294 | 7.96k | return u_terminateChars(originalDest, destCapacity, length, &errorCode); |
295 | 7.96k | } |
296 | | |
297 | | void |
298 | | UnicodeString::doCodepageCreate(const char *codepageData, |
299 | | int32_t dataLength, |
300 | | const char *codepage) |
301 | 2.54k | { |
302 | | // if there's nothing to convert, do nothing |
303 | 2.54k | if(codepageData == 0 || dataLength == 0 || dataLength < -1) { |
304 | 1 | return; |
305 | 1 | } |
306 | 2.54k | if(dataLength == -1) { |
307 | 0 | dataLength = (int32_t)uprv_strlen(codepageData); |
308 | 0 | } |
309 | | |
310 | 2.54k | UErrorCode status = U_ZERO_ERROR; |
311 | | |
312 | | // create the converter |
313 | | // if the codepage is the default, use our cache |
314 | | // if it is an empty string, then use the "invariant character" conversion |
315 | 2.54k | UConverter *converter; |
316 | 2.54k | if (codepage == 0) { |
317 | 0 | const char *defaultName = ucnv_getDefaultName(); |
318 | 0 | if(UCNV_FAST_IS_UTF8(defaultName)) { |
319 | 0 | setToUTF8(StringPiece(codepageData, dataLength)); |
320 | 0 | return; |
321 | 0 | } |
322 | 0 | converter = u_getDefaultConverter(&status); |
323 | 2.54k | } else if(*codepage == 0) { |
324 | | // use the "invariant characters" conversion |
325 | 0 | if(cloneArrayIfNeeded(dataLength, dataLength, false)) { |
326 | 0 | u_charsToUChars(codepageData, getArrayStart(), dataLength); |
327 | 0 | setLength(dataLength); |
328 | 0 | } else { |
329 | 0 | setToBogus(); |
330 | 0 | } |
331 | 0 | return; |
332 | 2.54k | } else { |
333 | 2.54k | converter = ucnv_open(codepage, &status); |
334 | 2.54k | } |
335 | | |
336 | | // if we failed, set the appropriate flags and return |
337 | 2.54k | if(U_FAILURE(status)) { |
338 | 15 | setToBogus(); |
339 | 15 | return; |
340 | 15 | } |
341 | | |
342 | | // perform the conversion |
343 | 2.52k | doCodepageCreate(codepageData, dataLength, converter, status); |
344 | 2.52k | if(U_FAILURE(status)) { |
345 | 0 | setToBogus(); |
346 | 0 | } |
347 | | |
348 | | // close the converter |
349 | 2.52k | if(codepage == 0) { |
350 | 0 | u_releaseDefaultConverter(converter); |
351 | 2.52k | } else { |
352 | 2.52k | ucnv_close(converter); |
353 | 2.52k | } |
354 | 2.52k | } |
355 | | |
356 | | void |
357 | | UnicodeString::doCodepageCreate(const char *codepageData, |
358 | | int32_t dataLength, |
359 | | UConverter *converter, |
360 | | UErrorCode &status) |
361 | 2.52k | { |
362 | 2.52k | if(U_FAILURE(status)) { |
363 | 0 | return; |
364 | 0 | } |
365 | | |
366 | | // set up the conversion parameters |
367 | 2.52k | const char *mySource = codepageData; |
368 | 2.52k | const char *mySourceEnd = mySource + dataLength; |
369 | 2.52k | char16_t *array, *myTarget; |
370 | | |
371 | | // estimate the size needed: |
372 | 2.52k | int32_t arraySize; |
373 | 2.52k | if(dataLength <= US_STACKBUF_SIZE) { |
374 | | // try to use the stack buffer |
375 | 1.59k | arraySize = US_STACKBUF_SIZE; |
376 | 1.59k | } else { |
377 | | // 1.25 char16_t's per source byte should cover most cases |
378 | 930 | arraySize = dataLength + (dataLength >> 2); |
379 | 930 | } |
380 | | |
381 | | // we do not care about the current contents |
382 | 2.52k | UBool doCopyArray = false; |
383 | 2.52k | for(;;) { |
384 | 2.52k | if(!cloneArrayIfNeeded(arraySize, arraySize, doCopyArray)) { |
385 | 0 | setToBogus(); |
386 | 0 | break; |
387 | 0 | } |
388 | | |
389 | | // perform the conversion |
390 | 2.52k | array = getArrayStart(); |
391 | 2.52k | myTarget = array + length(); |
392 | 2.52k | ucnv_toUnicode(converter, &myTarget, array + getCapacity(), |
393 | 2.52k | &mySource, mySourceEnd, 0, true, &status); |
394 | | |
395 | | // update the conversion parameters |
396 | 2.52k | setLength((int32_t)(myTarget - array)); |
397 | | |
398 | | // allocate more space and copy data, if needed |
399 | 2.52k | if(status == U_BUFFER_OVERFLOW_ERROR) { |
400 | | // reset the error code |
401 | 0 | status = U_ZERO_ERROR; |
402 | | |
403 | | // keep the previous conversion results |
404 | 0 | doCopyArray = true; |
405 | | |
406 | | // estimate the new size needed, larger than before |
407 | | // try 2 char16_t's per remaining source byte |
408 | 0 | arraySize = (int32_t)(length() + 2 * (mySourceEnd - mySource)); |
409 | 2.52k | } else { |
410 | 2.52k | break; |
411 | 2.52k | } |
412 | 2.52k | } |
413 | 2.52k | } |
414 | | |
415 | | U_NAMESPACE_END |
416 | | |
417 | | #endif |