/src/gpac/src/utils/utf.c
Line | Count | Source |
1 | | /* |
2 | | * GPAC - Multimedia Framework C SDK |
3 | | * |
4 | | * Authors: Jean Le Feuvre |
5 | | * Copyright (c) Telecom ParisTech 2007-2022 |
6 | | * All rights reserved |
7 | | * |
8 | | * This file is part of GPAC / common tools sub-project |
9 | | * |
10 | | * GPAC is free software; you can redistribute it and/or modify |
11 | | * it under the terms of the GNU Lesser General Public License as published by |
12 | | * the Free Software Foundation; either version 2, or (at your option) |
13 | | * any later version. |
14 | | * |
15 | | * GPAC is distributed in the hope that it will be useful, |
16 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
17 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
18 | | * GNU Lesser General Public License for more details. |
19 | | * |
20 | | * You should have received a copy of the GNU Lesser General Public |
21 | | * License along with this library; see the file COPYING. If not, write to |
22 | | * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. |
23 | | * |
24 | | */ |
25 | | |
26 | | |
27 | | #include <gpac/utf.h> |
28 | | |
29 | | |
30 | | #if 1 |
31 | | |
32 | | |
33 | | /* |
34 | | * Copyright 2001-2004 Unicode, Inc. |
35 | | * |
36 | | * Disclaimer |
37 | | * |
38 | | * This source code is provided as is by Unicode, Inc. No claims are |
39 | | * made as to fitness for any particular purpose. No warranties of any |
40 | | * kind are expressed or implied. The recipient agrees to determine |
41 | | * applicability of information provided. If this file has been |
42 | | * purchased on magnetic or optical media from Unicode, Inc., the |
43 | | * sole remedy for any claim will be exchange of defective media |
44 | | * within 90 days of receipt. |
45 | | * |
46 | | * Limitations on Rights to Redistribute This Code |
47 | | * |
48 | | * Unicode, Inc. hereby grants the right to freely use the information |
49 | | * supplied in this file in the creation of products supporting the |
50 | | * Unicode Standard, and to make copies of this file in any form |
51 | | * for internal or external distribution as long as this notice |
52 | | * remains attached. |
53 | | */ |
54 | | |
55 | | /* --------------------------------------------------------------------- |
56 | | |
57 | | Conversions between UTF32, UTF-16, and UTF-8. Source code file. |
58 | | Author: Mark E. Davis, 1994. |
59 | | Rev History: Rick McGowan, fixes & updates May 2001. |
60 | | Sept 2001: fixed const & error conditions per |
61 | | mods suggested by S. Parent & A. Lillich. |
62 | | June 2002: Tim Dodd added detection and handling of incomplete |
63 | | source sequences, enhanced error detection, added casts |
64 | | to eliminate compiler warnings. |
65 | | July 2003: slight mods to back out aggressive FFFE detection. |
66 | | Jan 2004: updated switches in from-UTF8 conversions. |
67 | | Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. |
68 | | |
69 | | See the header file "ConvertUTF.h" for complete documentation. |
70 | | |
71 | | ------------------------------------------------------------------------ */ |
72 | | |
73 | | typedef u32 UTF32; /* at least 32 bits */ |
74 | | typedef u16 UTF16; /* at least 16 bits */ |
75 | | typedef u8 UTF8; /* typically 8 bits */ |
76 | | typedef u8 Boolean; /* 0 or 1 */ |
77 | | |
78 | | /* Some fundamental constants */ |
79 | 0 | #define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD |
80 | 0 | #define UNI_MAX_BMP (UTF32)0x0000FFFF |
81 | 0 | #define UNI_MAX_UTF16 (UTF32)0x0010FFFF |
82 | | #define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF |
83 | | #define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF |
84 | | |
85 | | typedef enum { |
86 | | conversionOK, /* conversion successful */ |
87 | | sourceExhausted, /* partial character in source, but hit end */ |
88 | | targetExhausted, /* insuff. room in target for conversion */ |
89 | | sourceIllegal /* source sequence is illegal/malformed */ |
90 | | } ConversionResult; |
91 | | |
92 | | typedef enum { |
93 | | strictConversion = 0, |
94 | | lenientConversion |
95 | | } ConversionFlags; |
96 | | |
97 | | static const int halfShift = 10; /* used for shifting by 10 bits */ |
98 | | |
99 | | static const UTF32 halfBase = 0x0010000UL; |
100 | | static const UTF32 halfMask = 0x3FFUL; |
101 | | |
102 | 0 | #define UNI_SUR_HIGH_START (UTF32)0xD800 |
103 | 0 | #define UNI_SUR_HIGH_END (UTF32)0xDBFF |
104 | 0 | #define UNI_SUR_LOW_START (UTF32)0xDC00 |
105 | 0 | #define UNI_SUR_LOW_END (UTF32)0xDFFF |
106 | | #undef false |
107 | | #undef true |
108 | 0 | #define false 0 |
109 | 0 | #define true 1 |
110 | | |
111 | | /* |
112 | | * Index into the table below with the first byte of a UTF-8 sequence to |
113 | | * get the number of trailing bytes that are supposed to follow it. |
114 | | * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is |
115 | | * left as-is for anyone who may want to do such conversion, which was |
116 | | * allowed in earlier algorithms. |
117 | | */ |
118 | | static const char trailingBytesForUTF8[256] = { |
119 | | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
120 | | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
121 | | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
122 | | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
123 | | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
124 | | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
125 | | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, |
126 | | 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 |
127 | | }; |
128 | | |
129 | | /* |
130 | | * Magic values subtracted from a buffer value during UTF8 conversion. |
131 | | * This table contains as many values as there might be trailing bytes |
132 | | * in a UTF-8 sequence. |
133 | | */ |
134 | | static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, |
135 | | 0x03C82080UL, 0xFA082080UL, 0x82082080UL |
136 | | }; |
137 | | |
138 | | /* |
139 | | * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed |
140 | | * into the first byte, depending on how many bytes follow. There are |
141 | | * as many entries in this table as there are UTF-8 sequence types. |
142 | | * (I.e., one byte sequence, two byte... etc.). Remember that sequencs |
143 | | * for *legal* UTF-8 will be 4 or fewer bytes total. |
144 | | */ |
145 | | static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; |
146 | | |
147 | | /* --------------------------------------------------------------------- */ |
148 | | |
149 | | /* The interface converts a whole buffer to avoid function-call overhead. |
150 | | * Constants have been gathered. Loops & conditionals have been removed as |
151 | | * much as possible for efficiency, in favor of drop-through switches. |
152 | | * (See "Note A" at the bottom of the file for equivalent code.) |
153 | | * If your compiler supports it, the "isLegalUTF8" call can be turned |
154 | | * into an inline function. |
155 | | */ |
156 | | |
157 | | /* --------------------------------------------------------------------- */ |
158 | | |
159 | | ConversionResult ConvertUTF16toUTF8 ( |
160 | | const UTF16** sourceStart, const UTF16* sourceEnd, |
161 | 0 | UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { |
162 | 0 | ConversionResult result = conversionOK; |
163 | 0 | const UTF16* source = *sourceStart; |
164 | 0 | UTF8* target = *targetStart; |
165 | 0 | while (source < sourceEnd) { |
166 | 0 | UTF32 ch; |
167 | 0 | unsigned short bytesToWrite = 0; |
168 | 0 | const UTF32 byteMask = 0xBF; |
169 | 0 | const UTF32 byteMark = 0x80; |
170 | 0 | const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ |
171 | 0 | ch = *source++; |
172 | | /* If we have a surrogate pair, convert to UTF32 first. */ |
173 | 0 | if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { |
174 | | /* If the 16 bits following the high surrogate are in the source buffer... */ |
175 | 0 | if (source < sourceEnd) { |
176 | 0 | UTF32 ch2 = *source; |
177 | | /* If it's a low surrogate, convert to UTF32. */ |
178 | 0 | if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { |
179 | 0 | ch = ((ch - UNI_SUR_HIGH_START) << halfShift) |
180 | 0 | + (ch2 - UNI_SUR_LOW_START) + halfBase; |
181 | 0 | ++source; |
182 | 0 | } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ |
183 | 0 | --source; /* return to the illegal value itself */ |
184 | 0 | result = sourceIllegal; |
185 | 0 | break; |
186 | 0 | } |
187 | 0 | } else { /* We don't have the 16 bits following the high surrogate. */ |
188 | 0 | --source; /* return to the high surrogate */ |
189 | 0 | result = sourceExhausted; |
190 | 0 | break; |
191 | 0 | } |
192 | 0 | } else if (flags == strictConversion) { |
193 | | /* UTF-16 surrogate values are illegal in UTF-32 */ |
194 | 0 | if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { |
195 | 0 | --source; /* return to the illegal value itself */ |
196 | 0 | result = sourceIllegal; |
197 | 0 | break; |
198 | 0 | } |
199 | 0 | } |
200 | | /* Figure out how many bytes the result will require */ |
201 | 0 | if (ch < (UTF32)0x80) { |
202 | 0 | bytesToWrite = 1; |
203 | 0 | } else if (ch < (UTF32)0x800) { |
204 | 0 | bytesToWrite = 2; |
205 | 0 | } else if (ch < (UTF32)0x10000) { |
206 | 0 | bytesToWrite = 3; |
207 | 0 | } else if (ch < (UTF32)0x110000) { |
208 | 0 | bytesToWrite = 4; |
209 | 0 | } else { |
210 | 0 | bytesToWrite = 3; |
211 | 0 | ch = UNI_REPLACEMENT_CHAR; |
212 | 0 | } |
213 | |
|
214 | 0 | target += bytesToWrite; |
215 | 0 | if (target > targetEnd) { |
216 | 0 | source = oldSource; /* Back up source pointer! */ |
217 | 0 | target -= bytesToWrite; |
218 | 0 | result = targetExhausted; |
219 | 0 | break; |
220 | 0 | } |
221 | 0 | switch (bytesToWrite) { /* note: everything falls through. */ |
222 | 0 | case 4: |
223 | 0 | *--target = (UTF8)((ch | byteMark) & byteMask); |
224 | 0 | ch >>= 6; |
225 | 0 | case 3: |
226 | 0 | *--target = (UTF8)((ch | byteMark) & byteMask); |
227 | 0 | ch >>= 6; |
228 | 0 | case 2: |
229 | 0 | *--target = (UTF8)((ch | byteMark) & byteMask); |
230 | 0 | ch >>= 6; |
231 | 0 | case 1: |
232 | 0 | *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); |
233 | 0 | } |
234 | 0 | target += bytesToWrite; |
235 | 0 | } |
236 | 0 | *sourceStart = source; |
237 | 0 | *targetStart = target; |
238 | 0 | return result; |
239 | 0 | } |
240 | | |
241 | | /* |
242 | | * Utility routine to tell whether a sequence of bytes is legal UTF-8. |
243 | | * This must be called with the length pre-determined by the first byte. |
244 | | * If not calling this from ConvertUTF8to*, then the length can be set by: |
245 | | * length = trailingBytesForUTF8[*source]+1; |
246 | | * and the sequence is illegal right away if there aren't that many bytes |
247 | | * available. |
248 | | * If presented with a length > 4, this returns false. The Unicode |
249 | | * definition of UTF-8 goes up to 4-byte sequences. |
250 | | */ |
251 | | |
252 | 0 | Boolean isLegalUTF8(const UTF8 *source, int length) { |
253 | 0 | UTF8 a; |
254 | 0 | const UTF8 *srcptr = source+length; |
255 | 0 | switch (length) { |
256 | 0 | default: |
257 | 0 | return false; |
258 | | /* Everything else falls through when "true"... */ |
259 | 0 | case 4: |
260 | 0 | if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; |
261 | 0 | case 3: |
262 | 0 | if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; |
263 | 0 | case 2: |
264 | 0 | if ((a = (*--srcptr)) > 0xBF) return false; |
265 | | |
266 | 0 | switch (*source) { |
267 | | /* no fall-through in this inner switch */ |
268 | 0 | case 0xE0: |
269 | 0 | if (a < 0xA0) return false; |
270 | 0 | break; |
271 | 0 | case 0xED: |
272 | 0 | if (a > 0x9F) return false; |
273 | 0 | break; |
274 | 0 | case 0xF0: |
275 | 0 | if (a < 0x90) return false; |
276 | 0 | break; |
277 | 0 | case 0xF4: |
278 | 0 | if (a > 0x8F) return false; |
279 | 0 | break; |
280 | 0 | default: |
281 | 0 | if (a < 0x80) return false; |
282 | 0 | } |
283 | | |
284 | 0 | case 1: |
285 | 0 | if (*source >= 0x80 && *source < 0xC2) return false; |
286 | 0 | } |
287 | 0 | if (*source > 0xF4) return false; |
288 | 0 | return true; |
289 | 0 | } |
290 | | |
291 | | /* --------------------------------------------------------------------- */ |
292 | | |
293 | | ConversionResult ConvertUTF8toUTF16 ( |
294 | | const UTF8** sourceStart, const UTF8* sourceEnd, |
295 | 0 | UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { |
296 | 0 | ConversionResult result = conversionOK; |
297 | 0 | const UTF8* source = *sourceStart; |
298 | 0 | UTF16* target = *targetStart; |
299 | 0 | while (source < sourceEnd) { |
300 | 0 | UTF32 ch = 0; |
301 | 0 | unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; |
302 | 0 | if (source + extraBytesToRead >= sourceEnd) { |
303 | 0 | result = sourceExhausted; |
304 | 0 | break; |
305 | 0 | } |
306 | | /* Do this check whether lenient or strict */ |
307 | 0 | if (! isLegalUTF8(source, extraBytesToRead+1)) { |
308 | 0 | result = sourceIllegal; |
309 | 0 | break; |
310 | 0 | } |
311 | | /* |
312 | | * The cases all fall through. See "Note A" below. |
313 | | */ |
314 | 0 | switch (extraBytesToRead) { |
315 | 0 | case 5: |
316 | 0 | ch += *source++; |
317 | 0 | ch <<= 6; /* remember, illegal UTF-8 */ |
318 | 0 | case 4: |
319 | 0 | ch += *source++; |
320 | 0 | ch <<= 6; /* remember, illegal UTF-8 */ |
321 | 0 | case 3: |
322 | 0 | ch += *source++; |
323 | 0 | ch <<= 6; |
324 | 0 | case 2: |
325 | 0 | ch += *source++; |
326 | 0 | ch <<= 6; |
327 | 0 | case 1: |
328 | 0 | ch += *source++; |
329 | 0 | ch <<= 6; |
330 | 0 | case 0: |
331 | 0 | ch += *source++; |
332 | 0 | } |
333 | 0 | ch -= offsetsFromUTF8[extraBytesToRead]; |
334 | |
|
335 | 0 | if (target >= targetEnd) { |
336 | 0 | source -= (extraBytesToRead+1); /* Back up source pointer! */ |
337 | 0 | result = targetExhausted; |
338 | 0 | break; |
339 | 0 | } |
340 | 0 | if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ |
341 | | /* UTF-16 surrogate values are illegal in UTF-32 */ |
342 | 0 | if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { |
343 | 0 | if (flags == strictConversion) { |
344 | 0 | source -= (extraBytesToRead+1); /* return to the illegal value itself */ |
345 | 0 | result = sourceIllegal; |
346 | 0 | break; |
347 | 0 | } else { |
348 | 0 | *target++ = UNI_REPLACEMENT_CHAR; |
349 | 0 | } |
350 | 0 | } else { |
351 | 0 | *target++ = (UTF16)ch; /* normal case */ |
352 | 0 | } |
353 | 0 | } else if (ch > UNI_MAX_UTF16) { |
354 | 0 | if (flags == strictConversion) { |
355 | 0 | result = sourceIllegal; |
356 | 0 | source -= (extraBytesToRead+1); /* return to the start */ |
357 | 0 | break; /* Bail out; shouldn't continue */ |
358 | 0 | } else { |
359 | 0 | *target++ = UNI_REPLACEMENT_CHAR; |
360 | 0 | } |
361 | 0 | } else { |
362 | | /* target is a character in range 0xFFFF - 0x10FFFF. */ |
363 | 0 | if (target + 1 >= targetEnd) { |
364 | 0 | source -= (extraBytesToRead+1); /* Back up source pointer! */ |
365 | 0 | result = targetExhausted; |
366 | 0 | break; |
367 | 0 | } |
368 | 0 | ch -= halfBase; |
369 | 0 | *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); |
370 | 0 | *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); |
371 | 0 | } |
372 | 0 | } |
373 | 0 | *sourceStart = source; |
374 | 0 | *targetStart = target; |
375 | 0 | return result; |
376 | 0 | } |
377 | | |
378 | | |
379 | | GF_EXPORT |
380 | | Bool gf_utf8_is_legal(const u8 *data, u32 length) |
381 | 0 | { |
382 | | //we simply run ConvertUTF8toUTF16 without target |
383 | 0 | const UTF8** sourceStart = (const UTF8**) &data; |
384 | 0 | const UTF8* sourceEnd = (const UTF8*) ( data + length ); |
385 | 0 | ConversionResult result = conversionOK; |
386 | 0 | const UTF8* source = *sourceStart; |
387 | |
|
388 | 0 | while (source < sourceEnd) { |
389 | 0 | UTF32 ch = 0; |
390 | 0 | unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; |
391 | 0 | if (source + extraBytesToRead >= sourceEnd) { |
392 | 0 | result = sourceExhausted; |
393 | 0 | break; |
394 | 0 | } |
395 | | /* Do this check whether lenient or strict */ |
396 | 0 | if (! isLegalUTF8(source, extraBytesToRead+1)) { |
397 | 0 | result = sourceIllegal; |
398 | 0 | break; |
399 | 0 | } |
400 | | /* |
401 | | * The cases all fall through. See "Note A" below. |
402 | | */ |
403 | 0 | switch (extraBytesToRead) { |
404 | 0 | case 5: |
405 | 0 | ch += *source++; |
406 | 0 | ch <<= 6; /* remember, illegal UTF-8 */ |
407 | 0 | case 4: |
408 | 0 | ch += *source++; |
409 | 0 | ch <<= 6; /* remember, illegal UTF-8 */ |
410 | 0 | case 3: |
411 | 0 | ch += *source++; |
412 | 0 | ch <<= 6; |
413 | 0 | case 2: |
414 | 0 | ch += *source++; |
415 | 0 | ch <<= 6; |
416 | 0 | case 1: |
417 | 0 | ch += *source++; |
418 | 0 | ch <<= 6; |
419 | 0 | case 0: |
420 | 0 | ch += *source++; |
421 | 0 | } |
422 | 0 | ch -= offsetsFromUTF8[extraBytesToRead]; |
423 | |
|
424 | 0 | if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ |
425 | | /* UTF-16 surrogate values are illegal in UTF-32 */ |
426 | 0 | if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { |
427 | 0 | result = sourceIllegal; |
428 | 0 | break; |
429 | 0 | } |
430 | 0 | } else if (ch > UNI_MAX_UTF16) { |
431 | 0 | result = sourceIllegal; |
432 | 0 | break; /* Bail out; shouldn't continue */ |
433 | 0 | } |
434 | 0 | } |
435 | 0 | return (result==conversionOK) ? GF_TRUE : GF_FALSE; |
436 | 0 | } |
437 | | |
438 | | GF_EXPORT |
439 | | u32 gf_utf8_wcslen (const unsigned short *s) |
440 | 0 | { |
441 | 0 | const unsigned short* ptr; |
442 | 0 | if (!s) return 0; |
443 | 0 | for (ptr = s; *ptr != (unsigned short)'\0'; ptr++) { |
444 | 0 | } |
445 | 0 | return (u32) ( ptr - s ); |
446 | 0 | } |
447 | | |
448 | | GF_EXPORT |
449 | | u32 gf_utf8_wcstombs(char* dest, size_t len, const unsigned short** srcp) |
450 | 0 | { |
451 | 0 | if (!srcp || !*srcp) |
452 | 0 | return 0; |
453 | 0 | else { |
454 | 0 | const UTF16** sourceStart = srcp; |
455 | 0 | const UTF16* sourceEnd = *srcp + gf_utf8_wcslen(*srcp); |
456 | 0 | UTF8* targetStart = (UTF8*) dest; |
457 | 0 | UTF8* targetEnd = (UTF8*) dest + len; |
458 | 0 | ConversionFlags flags = strictConversion; |
459 | |
|
460 | 0 | ConversionResult res = ConvertUTF16toUTF8(sourceStart, sourceEnd, &targetStart, targetEnd, flags); |
461 | 0 | if (res != conversionOK) return GF_UTF8_FAIL; |
462 | 0 | if (targetStart >= targetEnd) return GF_UTF8_FAIL; |
463 | 0 | *targetStart = 0; |
464 | 0 | *srcp=NULL; |
465 | 0 | return (u32) strlen(dest); |
466 | 0 | } |
467 | 0 | } |
468 | | |
469 | | GF_EXPORT |
470 | | u32 gf_utf8_mbstowcs(unsigned short* dest, size_t len, const char** srcp) |
471 | 0 | { |
472 | 0 | if (!srcp || !*srcp) |
473 | 0 | return 0; |
474 | 0 | else { |
475 | 0 | const UTF8** sourceStart = (const UTF8**) srcp; |
476 | 0 | const UTF8* sourceEnd = (const UTF8*) ( *srcp + strlen( *srcp) ); |
477 | 0 | UTF16* targetStart = (UTF16* ) dest; |
478 | 0 | UTF16* targetEnd = (UTF16* ) (dest + len); |
479 | 0 | ConversionFlags flags = strictConversion; |
480 | 0 | ConversionResult res = ConvertUTF8toUTF16(sourceStart, sourceEnd, &targetStart, targetEnd, flags); |
481 | 0 | if (res != conversionOK) return GF_UTF8_FAIL; |
482 | 0 | if (targetStart >= targetEnd) return GF_UTF8_FAIL; |
483 | 0 | *targetStart = 0; |
484 | 0 | *srcp=NULL; |
485 | 0 | return gf_utf8_wcslen(dest); |
486 | 0 | } |
487 | 0 | } |
488 | | |
489 | | |
490 | | #else |
491 | | |
492 | | GF_EXPORT |
493 | | u32 gf_utf8_wcslen (const unsigned short *s) |
494 | | { |
495 | | const unsigned short* ptr; |
496 | | for (ptr = s; *ptr != (unsigned short)'\0'; ptr++) { |
497 | | } |
498 | | return (u32) (ptr - s); |
499 | | } |
500 | | |
501 | | GF_EXPORT |
502 | | u32 gf_utf8_wcstombs(char* dest, size_t len, const unsigned short** srcp) |
503 | | { |
504 | | /* |
505 | | * Original code from the GNU UTF-8 Library |
506 | | */ |
507 | | size_t count; |
508 | | const unsigned short * src = *srcp; |
509 | | |
510 | | if (dest != NULL) { |
511 | | char* destptr = dest; |
512 | | for (;; src++) { |
513 | | unsigned char c; |
514 | | unsigned short wc = *src; |
515 | | if (wc < 0x80) { |
516 | | if (wc == (wchar_t)'\0') { |
517 | | if (len == 0) { |
518 | | *srcp = src; |
519 | | break; |
520 | | } |
521 | | *destptr = '\0'; |
522 | | *srcp = NULL; |
523 | | break; |
524 | | } |
525 | | count = 0; |
526 | | c = (unsigned char) wc; |
527 | | } else if (wc < 0x800) { |
528 | | count = 1; |
529 | | c = (unsigned char) ((wc >> 6) | 0xC0); |
530 | | } else { |
531 | | count = 2; |
532 | | c = (unsigned char) ((wc >> 12) | 0xE0); |
533 | | } |
534 | | if (len <= count) { |
535 | | *srcp = src; |
536 | | break; |
537 | | } |
538 | | len -= count+1; |
539 | | *destptr++ = c; |
540 | | if (count > 0) |
541 | | do { |
542 | | *destptr++ = (unsigned char)(((wc >> (6 * --count)) & 0x3F) | 0x80); |
543 | | } while (count > 0); |
544 | | } |
545 | | return (u32) (destptr - dest); |
546 | | } else { |
547 | | /* Ignore dest and len. */ |
548 | | size_t totalcount = 0; |
549 | | for (;; src++) { |
550 | | unsigned short wc = *src; |
551 | | size_t count; |
552 | | if (wc < 0x80) { |
553 | | if (wc == (wchar_t)'\0') { |
554 | | *srcp = NULL; |
555 | | break; |
556 | | } |
557 | | count = 1; |
558 | | } else if (wc < 0x800) { |
559 | | count = 2; |
560 | | } else { |
561 | | count = 3; |
562 | | } |
563 | | totalcount += count; |
564 | | } |
565 | | return (u32) totalcount; |
566 | | } |
567 | | } |
568 | | |
569 | | |
570 | | typedef struct |
571 | | { |
572 | | u32 count : 16; /* number of bytes remaining to be processed */ |
573 | | u32 value : 16; /* if count > 0: partial wide character */ |
574 | | /* |
575 | | If WCHAR_T_BITS == 16, need 2 bits for count, |
576 | | 12 bits for value (10 for mbstowcs direction, 12 for wcstombs direction). |
577 | | */ |
578 | | } gf_utf8_mbstate_t; |
579 | | |
580 | | static gf_utf8_mbstate_t internal; |
581 | | |
582 | | GF_EXPORT |
583 | | u32 gf_utf8_mbstowcs(unsigned short* dest, size_t len, const char** srcp) |
584 | | { |
585 | | gf_utf8_mbstate_t* ps = &internal; |
586 | | const char *src = *srcp; |
587 | | |
588 | | unsigned short* destptr = dest; |
589 | | for (; len > 0; destptr++, len--) { |
590 | | const char* backup_src = src; |
591 | | unsigned char c; |
592 | | unsigned short wc; |
593 | | size_t count; |
594 | | if (ps->count == 0) { |
595 | | c = (unsigned char) *src; |
596 | | if (c < 0x80) { |
597 | | *destptr = (wchar_t) c; |
598 | | if (c == 0) { |
599 | | src = NULL; |
600 | | break; |
601 | | } |
602 | | src++; |
603 | | continue; |
604 | | } else if (c < 0xC0) { |
605 | | /* Spurious 10XXXXXX byte is invalid. */ |
606 | | goto bad_input; |
607 | | } |
608 | | if (c < 0xE0) { |
609 | | wc = (wchar_t)(c & 0x1F) << 6; |
610 | | count = 1; |
611 | | if (c < 0xC2) goto bad_input; |
612 | | } else if (c < 0xF0) { |
613 | | wc = (wchar_t)(c & 0x0F) << 12; |
614 | | count = 2; |
615 | | } |
616 | | else goto bad_input; |
617 | | src++; |
618 | | } else { |
619 | | wc = ps->value << 6; |
620 | | count = ps->count; |
621 | | } |
622 | | for (;;) { |
623 | | c = (unsigned char) *src++ ^ 0x80; |
624 | | if (!(c < 0x40)) goto bad_input_backup; |
625 | | wc |= (unsigned short) c << (6 * --count); |
626 | | if (count == 0) |
627 | | break; |
628 | | /* The following test is only necessary once for every character, |
629 | | but it would be too complicated to perform it once only, on |
630 | | the first pass through this loop. */ |
631 | | if ((unsigned short) wc < ((unsigned short) 1 << (5 * count + 6))) |
632 | | goto bad_input_backup; |
633 | | } |
634 | | *destptr = wc; |
635 | | ps->count = 0; |
636 | | continue; |
637 | | |
638 | | bad_input_backup: |
639 | | src = backup_src; |
640 | | goto bad_input; |
641 | | } |
642 | | *srcp = src; |
643 | | return (u32) (destptr - dest); |
644 | | |
645 | | bad_input: |
646 | | *srcp = src; |
647 | | return GF_UTF8_FAIL; |
648 | | } |
649 | | |
650 | | |
651 | | #endif |
652 | | |
653 | | |
654 | | GF_EXPORT |
655 | | GF_Err gf_utf_get_string_from_bom(const u8 *data, u32 size, char **out_ptr, char **result, u32 *res_size) |
656 | 0 | { |
657 | 0 | u32 unicode_type = 0; |
658 | 0 | if (!out_ptr || !result || !data) return GF_BAD_PARAM; |
659 | 0 | *out_ptr = NULL; |
660 | 0 | *result = (char *) data; |
661 | 0 | if (res_size) *res_size = size; |
662 | |
|
663 | 0 | if (size>=5) { |
664 | | /*0: no unicode, 1: UTF-16BE, 2: UTF-16LE*/ |
665 | 0 | if ((data[0]==0xFF) && (data[1]==0xFE)) { |
666 | 0 | if (!data[2] && !data[3]) { |
667 | 0 | return GF_OK; |
668 | 0 | } else { |
669 | 0 | unicode_type = 2; |
670 | 0 | } |
671 | 0 | } else if ((data[0]==0xFE) && (data[1]==0xFF)) { |
672 | 0 | if (!data[2] && !data[3]) { |
673 | 0 | return GF_OK; |
674 | 0 | } else { |
675 | 0 | unicode_type = 1; |
676 | 0 | } |
677 | 0 | } else if ((data[0]==0xEF) && (data[1]==0xBB) && (data[2]==0xBF)) { |
678 | 0 | if (res_size) *res_size = size-4; |
679 | 0 | *result = (char *) (data+4); |
680 | 0 | return GF_OK; |
681 | 0 | } |
682 | 0 | } |
683 | | |
684 | 0 | if (!unicode_type) { |
685 | 0 | *result = (char *) data; |
686 | 0 | return GF_OK; |
687 | 0 | } |
688 | | |
689 | 0 | if (size%2) size--; |
690 | 0 | u16 *str_wc = gf_malloc(size+2); |
691 | 0 | if (!str_wc) return GF_OUT_OF_MEM; |
692 | 0 | u16 *srcwc; |
693 | 0 | char *dst = gf_malloc(size+2); |
694 | 0 | if (!dst) { |
695 | 0 | gf_free(str_wc); |
696 | 0 | return GF_OUT_OF_MEM; |
697 | 0 | } |
698 | 0 | *out_ptr = dst; |
699 | 0 | u32 i; |
700 | 0 | for (i=0; i<size; i+=2) { |
701 | 0 | u16 wchar=0; |
702 | 0 | u8 c1 = data[i]; |
703 | 0 | u8 c2 = data[i+1]; |
704 | | |
705 | | /*Little-endian order*/ |
706 | 0 | if (unicode_type==2) { |
707 | 0 | if (c2) { |
708 | 0 | wchar = c2; |
709 | 0 | wchar <<=8; |
710 | 0 | wchar |= c1; |
711 | 0 | } |
712 | 0 | else wchar = c1; |
713 | 0 | } else { |
714 | 0 | wchar = c1; |
715 | 0 | if (c2) { |
716 | 0 | wchar <<= 8; |
717 | 0 | wchar |= c2; |
718 | 0 | } |
719 | 0 | } |
720 | 0 | str_wc[i/2] = wchar; |
721 | 0 | } |
722 | 0 | str_wc[i/2] = 0; |
723 | 0 | srcwc = str_wc; |
724 | 0 | u32 res = gf_utf8_wcstombs(dst, size, (const unsigned short **) &srcwc); |
725 | 0 | gf_free(str_wc); |
726 | 0 | if (res==GF_UTF8_FAIL) { |
727 | 0 | gf_free(dst); |
728 | 0 | *out_ptr = NULL; |
729 | 0 | return GF_IO_ERR; |
730 | 0 | } |
731 | 0 | *result = dst; |
732 | 0 | if (res_size) *res_size = res+1; |
733 | 0 | return GF_OK; |
734 | 0 | } |
735 | | |
736 | | |
737 | | #if defined(WIN32) |
738 | | |
739 | | GF_EXPORT |
740 | | wchar_t* gf_utf8_to_wcs(const char* str) |
741 | | { |
742 | | size_t source_len; |
743 | | wchar_t* result; |
744 | | if (str == 0) return 0; |
745 | | source_len = 1+strlen(str); |
746 | | result = gf_calloc(source_len, sizeof(wchar_t)); |
747 | | if (!result) |
748 | | return 0; |
749 | | if (gf_utf8_mbstowcs(result, source_len, &str) == GF_UTF8_FAIL) { |
750 | | gf_free(result); |
751 | | return 0; |
752 | | } |
753 | | return result; |
754 | | } |
755 | | |
756 | | GF_EXPORT |
757 | | char* gf_wcs_to_utf8(const wchar_t* str) |
758 | | { |
759 | | size_t source_len; |
760 | | char* result; |
761 | | if (str == 0) return 0; |
762 | | source_len = 1+wcslen(str); |
763 | | result = gf_calloc(source_len, UTF8_MAX_BYTES_PER_CHAR); |
764 | | if (!result) |
765 | | return 0; |
766 | | if (gf_utf8_wcstombs(result, source_len * UTF8_MAX_BYTES_PER_CHAR, &str) == GF_UTF8_FAIL) { |
767 | | gf_free(result); |
768 | | return 0; |
769 | | } |
770 | | return result; |
771 | | } |
772 | | #endif |