1 /*
2 * Copyright 2001-2004 Unicode, Inc.
3 *
4 * Disclaimer
5 *
6 * This source code is provided as is by Unicode, Inc. No claims are
7 * made as to fitness for any particular purpose. No warranties of any
8 * kind are expressed or implied. The recipient agrees to determine
9 * applicability of information provided. If this file has been
10 * purchased on magnetic or optical media from Unicode, Inc., the
11 * sole remedy for any claim will be exchange of defective media
12 * within 90 days of receipt.
13 *
14 * Limitations on Rights to Redistribute This Code
15 *
16 * Unicode, Inc. hereby grants the right to freely use the information
17 * supplied in this file in the creation of products supporting the
18 * Unicode Standard, and to make copies of this file in any form
19 * for internal or external distribution as long as this notice
20 * remains attached.
21 */
22
23 /* $Id: ConvertUTF.c 8178 2007-08-05 13:55:26Z leonardo $*/
24
25
26 /* ---------------------------------------------------------------------
27
28 Conversions between UTF32, UTF-16, and UTF-8. Source code file.
29 Author: Mark E. Davis, 1994.
30 Rev History: Rick McGowan, fixes & updates May 2001.
31 Sept 2001: fixed const & error conditions per
32 mods suggested by S. Parent & A. Lillich.
33 June 2002: Tim Dodd added detection and handling of incomplete
34 source sequences, enhanced error detection, added casts
35 to eliminate compiler warnings.
36 July 2003: slight mods to back out aggressive FFFE detection.
37 Jan 2004: updated switches in from-UTF8 conversions.
38 Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
39
40 See the header file "ConvertUTF.h" for complete documentation.
41
42 ------------------------------------------------------------------------ */
43
44
45 #include "ConvertUTF.h"
46 #ifdef CVTUTF_DEBUG
47 #include <stdio.h>
48 #endif
49
50 static const int halfShift = 10; /* used for shifting by 10 bits */
51
52 static const UTF32 halfBase = 0x0010000UL;
53 static const UTF32 halfMask = 0x3FFUL;
54
55 #define UNI_SUR_HIGH_START (UTF32)0xD800
56 #define UNI_SUR_HIGH_END (UTF32)0xDBFF
57 #define UNI_SUR_LOW_START (UTF32)0xDC00
58 #define UNI_SUR_LOW_END (UTF32)0xDFFF
59 #define false 0
60 #define true 1
61
62 /* --------------------------------------------------------------------- */
63
ConvertUTF32toUTF16(const UTF32 ** sourceStart,const UTF32 * sourceEnd,UTF16 ** targetStart,UTF16 * targetEnd,ConversionFlags flags)64 ConversionResult ConvertUTF32toUTF16 (
65 const UTF32** sourceStart, const UTF32* sourceEnd,
66 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
67 ConversionResult result = conversionOK;
68 const UTF32* source = *sourceStart;
69 UTF16* target = *targetStart;
70 while (source < sourceEnd) {
71 UTF32 ch;
72 if (target >= targetEnd) {
73 result = targetExhausted; break;
74 }
75 ch = *source++;
76 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
77 /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
78 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
79 if (flags == strictConversion) {
80 --source; /* return to the illegal value itself */
81 result = sourceIllegal;
82 break;
83 } else {
84 *target++ = UNI_REPLACEMENT_CHAR;
85 }
86 } else {
87 *target++ = (UTF16)ch; /* normal case */
88 }
89 } else if (ch > UNI_MAX_LEGAL_UTF32) {
90 if (flags == strictConversion) {
91 result = sourceIllegal;
92 } else {
93 *target++ = UNI_REPLACEMENT_CHAR;
94 }
95 } else {
96 /* target is a character in range 0xFFFF - 0x10FFFF. */
97 if (target + 1 >= targetEnd) {
98 --source; /* Back up source pointer! */
99 result = targetExhausted; break;
100 }
101 ch -= halfBase;
102 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
103 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
104 }
105 }
106 *sourceStart = source;
107 *targetStart = target;
108 return result;
109 }
110
111 /* --------------------------------------------------------------------- */
112
ConvertUTF16toUTF32(const UTF16 ** sourceStart,const UTF16 * sourceEnd,UTF32 ** targetStart,UTF32 * targetEnd,ConversionFlags flags)113 ConversionResult ConvertUTF16toUTF32 (
114 const UTF16** sourceStart, const UTF16* sourceEnd,
115 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
116 ConversionResult result = conversionOK;
117 const UTF16* source = *sourceStart;
118 UTF32* target = *targetStart;
119 UTF32 ch, ch2;
120 while (source < sourceEnd) {
121 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
122 ch = *source++;
123 /* If we have a surrogate pair, convert to UTF32 first. */
124 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
125 /* If the 16 bits following the high surrogate are in the source buffer... */
126 if (source < sourceEnd) {
127 ch2 = *source;
128 /* If it's a low surrogate, convert to UTF32. */
129 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
130 ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
131 + (ch2 - UNI_SUR_LOW_START) + halfBase;
132 ++source;
133 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
134 --source; /* return to the illegal value itself */
135 result = sourceIllegal;
136 break;
137 }
138 } else { /* We don't have the 16 bits following the high surrogate. */
139 --source; /* return to the high surrogate */
140 result = sourceExhausted;
141 break;
142 }
143 } else if (flags == strictConversion) {
144 /* UTF-16 surrogate values are illegal in UTF-32 */
145 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
146 --source; /* return to the illegal value itself */
147 result = sourceIllegal;
148 break;
149 }
150 }
151 if (target >= targetEnd) {
152 source = oldSource; /* Back up source pointer! */
153 result = targetExhausted; break;
154 }
155 *target++ = ch;
156 }
157 *sourceStart = source;
158 *targetStart = target;
159 #ifdef CVTUTF_DEBUG
160 if (result == sourceIllegal) {
161 fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
162 fflush(stderr);
163 }
164 #endif
165 return result;
166 }
167
168 /* --------------------------------------------------------------------- */
169
170 /*
171 * Index into the table below with the first byte of a UTF-8 sequence to
172 * get the number of trailing bytes that are supposed to follow it.
173 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
174 * left as-is for anyone who may want to do such conversion, which was
175 * allowed in earlier algorithms.
176 */
177 static const char trailingBytesForUTF8[256] = {
178 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
179 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
180 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
181 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
182 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
183 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
184 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
185 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
186 };
187
188 /*
189 * Magic values subtracted from a buffer value during UTF8 conversion.
190 * This table contains as many values as there might be trailing bytes
191 * in a UTF-8 sequence.
192 */
193 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
194 0x03C82080UL, 0xFA082080UL, 0x82082080UL };
195
196 /*
197 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
198 * into the first byte, depending on how many bytes follow. There are
199 * as many entries in this table as there are UTF-8 sequence types.
200 * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
201 * for *legal* UTF-8 will be 4 or fewer bytes total.
202 */
203 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
204
205 /* --------------------------------------------------------------------- */
206
207 /* The interface converts a whole buffer to avoid function-call overhead.
208 * Constants have been gathered. Loops & conditionals have been removed as
209 * much as possible for efficiency, in favor of drop-through switches.
210 * (See "Note A" at the bottom of the file for equivalent code.)
211 * If your compiler supports it, the "isLegalUTF8" call can be turned
212 * into an inline function.
213 */
214
215 /* --------------------------------------------------------------------- */
216
ConvertUTF16toUTF8(const UTF16 ** sourceStart,const UTF16 * sourceEnd,UTF8 ** targetStart,UTF8 * targetEnd,ConversionFlags flags)217 ConversionResult ConvertUTF16toUTF8 (
218 const UTF16** sourceStart, const UTF16* sourceEnd,
219 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
220 ConversionResult result = conversionOK;
221 const UTF16* source = *sourceStart;
222 UTF8* target = *targetStart;
223 while (source < sourceEnd) {
224 UTF32 ch;
225 unsigned short bytesToWrite = 0;
226 const UTF32 byteMask = 0xBF;
227 const UTF32 byteMark = 0x80;
228 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
229 ch = *source++;
230 /* If we have a surrogate pair, convert to UTF32 first. */
231 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
232 /* If the 16 bits following the high surrogate are in the source buffer... */
233 if (source < sourceEnd) {
234 UTF32 ch2 = *source;
235 /* If it's a low surrogate, convert to UTF32. */
236 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
237 ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
238 + (ch2 - UNI_SUR_LOW_START) + halfBase;
239 ++source;
240 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
241 --source; /* return to the illegal value itself */
242 result = sourceIllegal;
243 break;
244 }
245 } else { /* We don't have the 16 bits following the high surrogate. */
246 --source; /* return to the high surrogate */
247 result = sourceExhausted;
248 break;
249 }
250 } else if (flags == strictConversion) {
251 /* UTF-16 surrogate values are illegal in UTF-32 */
252 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
253 --source; /* return to the illegal value itself */
254 result = sourceIllegal;
255 break;
256 }
257 }
258 /* Figure out how many bytes the result will require */
259 if (ch < (UTF32)0x80) { bytesToWrite = 1;
260 } else if (ch < (UTF32)0x800) { bytesToWrite = 2;
261 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3;
262 } else if (ch < (UTF32)0x110000) { bytesToWrite = 4;
263 } else { bytesToWrite = 3;
264 ch = UNI_REPLACEMENT_CHAR;
265 }
266
267 target += bytesToWrite;
268 if (target > targetEnd) {
269 source = oldSource; /* Back up source pointer! */
270 target -= bytesToWrite; result = targetExhausted; break;
271 }
272 switch (bytesToWrite) { /* note: everything falls through. */
273 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
274 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
275 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
276 case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]);
277 }
278 target += bytesToWrite;
279 }
280 *sourceStart = source;
281 *targetStart = target;
282 return result;
283 }
284
285 /* --------------------------------------------------------------------- */
286
287 /*
288 * Utility routine to tell whether a sequence of bytes is legal UTF-8.
289 * This must be called with the length pre-determined by the first byte.
290 * If not calling this from ConvertUTF8to*, then the length can be set by:
291 * length = trailingBytesForUTF8[*source]+1;
292 * and the sequence is illegal right away if there aren't that many bytes
293 * available.
294 * If presented with a length > 4, this returns false. The Unicode
295 * definition of UTF-8 goes up to 4-byte sequences.
296 */
297
isLegalUTF8(const UTF8 * source,int length)298 static Boolean isLegalUTF8(const UTF8 *source, int length) {
299 UTF8 a;
300 const UTF8 *srcptr = source+length;
301 switch (length) {
302 default: return false;
303 /* Everything else falls through when "true"... */
304 case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
305 case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
306 case 2: if ((a = (*--srcptr)) > 0xBF) return false;
307
308 switch (*source) {
309 /* no fall-through in this inner switch */
310 case 0xE0: if (a < 0xA0) return false; break;
311 case 0xED: if (a > 0x9F) return false; break;
312 case 0xF0: if (a < 0x90) return false; break;
313 case 0xF4: if (a > 0x8F) return false; break;
314 default: if (a < 0x80) return false;
315 }
316
317 case 1: if (*source >= 0x80 && *source < 0xC2) return false;
318 }
319 if (*source > 0xF4) return false;
320 return true;
321 }
322
323 /* --------------------------------------------------------------------- */
324
325 /*
326 * Exported function to return whether a UTF-8 sequence is legal or not.
327 * This is not used here; it's just exported.
328 */
isLegalUTF8Sequence(const UTF8 * source,const UTF8 * sourceEnd)329 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
330 int length = trailingBytesForUTF8[*source]+1;
331 if (source+length > sourceEnd) {
332 return false;
333 }
334 return isLegalUTF8(source, length);
335 }
336
337 /* --------------------------------------------------------------------- */
338
ConvertUTF8toUTF16(const UTF8 ** sourceStart,const UTF8 * sourceEnd,UTF16 ** targetStart,UTF16 * targetEnd,ConversionFlags flags)339 ConversionResult ConvertUTF8toUTF16 (
340 const UTF8** sourceStart, const UTF8* sourceEnd,
341 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
342 ConversionResult result = conversionOK;
343 const UTF8* source = *sourceStart;
344 UTF16* target = *targetStart;
345 while (source < sourceEnd) {
346 UTF32 ch = 0;
347 unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
348 if (source + extraBytesToRead >= sourceEnd) {
349 result = sourceExhausted; break;
350 }
351 /* Do this check whether lenient or strict */
352 if (! isLegalUTF8(source, extraBytesToRead+1)) {
353 result = sourceIllegal;
354 break;
355 }
356 /*
357 * The cases all fall through. See "Note A" below.
358 */
359 switch (extraBytesToRead) {
360 case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
361 case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
362 case 3: ch += *source++; ch <<= 6;
363 case 2: ch += *source++; ch <<= 6;
364 case 1: ch += *source++; ch <<= 6;
365 case 0: ch += *source++;
366 }
367 ch -= offsetsFromUTF8[extraBytesToRead];
368
369 if (target >= targetEnd) {
370 source -= (extraBytesToRead+1); /* Back up source pointer! */
371 result = targetExhausted; break;
372 }
373 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
374 /* UTF-16 surrogate values are illegal in UTF-32 */
375 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
376 if (flags == strictConversion) {
377 source -= (extraBytesToRead+1); /* return to the illegal value itself */
378 result = sourceIllegal;
379 break;
380 } else {
381 *target++ = UNI_REPLACEMENT_CHAR;
382 }
383 } else {
384 *target++ = (UTF16)ch; /* normal case */
385 }
386 } else if (ch > UNI_MAX_UTF16) {
387 if (flags == strictConversion) {
388 result = sourceIllegal;
389 source -= (extraBytesToRead+1); /* return to the start */
390 break; /* Bail out; shouldn't continue */
391 } else {
392 *target++ = UNI_REPLACEMENT_CHAR;
393 }
394 } else {
395 /* target is a character in range 0xFFFF - 0x10FFFF. */
396 if (target + 1 >= targetEnd) {
397 source -= (extraBytesToRead+1); /* Back up source pointer! */
398 result = targetExhausted; break;
399 }
400 ch -= halfBase;
401 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
402 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
403 }
404 }
405 *sourceStart = source;
406 *targetStart = target;
407 return result;
408 }
409
410 /* --------------------------------------------------------------------- */
411
ConvertUTF32toUTF8(const UTF32 ** sourceStart,const UTF32 * sourceEnd,UTF8 ** targetStart,UTF8 * targetEnd,ConversionFlags flags)412 ConversionResult ConvertUTF32toUTF8 (
413 const UTF32** sourceStart, const UTF32* sourceEnd,
414 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
415 ConversionResult result = conversionOK;
416 const UTF32* source = *sourceStart;
417 UTF8* target = *targetStart;
418 while (source < sourceEnd) {
419 UTF32 ch;
420 unsigned short bytesToWrite = 0;
421 const UTF32 byteMask = 0xBF;
422 const UTF32 byteMark = 0x80;
423 ch = *source++;
424 if (flags == strictConversion ) {
425 /* UTF-16 surrogate values are illegal in UTF-32 */
426 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
427 --source; /* return to the illegal value itself */
428 result = sourceIllegal;
429 break;
430 }
431 }
432 /*
433 * Figure out how many bytes the result will require. Turn any
434 * illegally large UTF32 things (> Plane 17) into replacement chars.
435 */
436 if (ch < (UTF32)0x80) { bytesToWrite = 1;
437 } else if (ch < (UTF32)0x800) { bytesToWrite = 2;
438 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3;
439 } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4;
440 } else { bytesToWrite = 3;
441 ch = UNI_REPLACEMENT_CHAR;
442 result = sourceIllegal;
443 }
444
445 target += bytesToWrite;
446 if (target > targetEnd) {
447 --source; /* Back up source pointer! */
448 target -= bytesToWrite; result = targetExhausted; break;
449 }
450 switch (bytesToWrite) { /* note: everything falls through. */
451 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
452 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
453 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
454 case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
455 }
456 target += bytesToWrite;
457 }
458 *sourceStart = source;
459 *targetStart = target;
460 return result;
461 }
462
463 /* --------------------------------------------------------------------- */
464
ConvertUTF8toUTF32(const UTF8 ** sourceStart,const UTF8 * sourceEnd,UTF32 ** targetStart,UTF32 * targetEnd,ConversionFlags flags)465 ConversionResult ConvertUTF8toUTF32 (
466 const UTF8** sourceStart, const UTF8* sourceEnd,
467 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
468 ConversionResult result = conversionOK;
469 const UTF8* source = *sourceStart;
470 UTF32* target = *targetStart;
471 while (source < sourceEnd) {
472 UTF32 ch = 0;
473 unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
474 if (source + extraBytesToRead >= sourceEnd) {
475 result = sourceExhausted; break;
476 }
477 /* Do this check whether lenient or strict */
478 if (! isLegalUTF8(source, extraBytesToRead+1)) {
479 result = sourceIllegal;
480 break;
481 }
482 /*
483 * The cases all fall through. See "Note A" below.
484 */
485 switch (extraBytesToRead) {
486 case 5: ch += *source++; ch <<= 6;
487 case 4: ch += *source++; ch <<= 6;
488 case 3: ch += *source++; ch <<= 6;
489 case 2: ch += *source++; ch <<= 6;
490 case 1: ch += *source++; ch <<= 6;
491 case 0: ch += *source++;
492 }
493 ch -= offsetsFromUTF8[extraBytesToRead];
494
495 if (target >= targetEnd) {
496 source -= (extraBytesToRead+1); /* Back up the source pointer! */
497 result = targetExhausted; break;
498 }
499 if (ch <= UNI_MAX_LEGAL_UTF32) {
500 /*
501 * UTF-16 surrogate values are illegal in UTF-32, and anything
502 * over Plane 17 (> 0x10FFFF) is illegal.
503 */
504 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
505 if (flags == strictConversion) {
506 source -= (extraBytesToRead+1); /* return to the illegal value itself */
507 result = sourceIllegal;
508 break;
509 } else {
510 *target++ = UNI_REPLACEMENT_CHAR;
511 }
512 } else {
513 *target++ = ch;
514 }
515 } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
516 result = sourceIllegal;
517 *target++ = UNI_REPLACEMENT_CHAR;
518 }
519 }
520 *sourceStart = source;
521 *targetStart = target;
522 return result;
523 }
524
525 /* ---------------------------------------------------------------------
526
527 Note A.
528 The fall-through switches in UTF-8 reading code save a
529 temp variable, some decrements & conditionals. The switches
530 are equivalent to the following loop:
531 {
532 int tmpBytesToRead = extraBytesToRead+1;
533 do {
534 ch += *source++;
535 --tmpBytesToRead;
536 if (tmpBytesToRead) ch <<= 6;
537 } while (tmpBytesToRead > 0);
538 }
539 In UTF-8 writing code, the switches on "bytesToWrite" are
540 similarly unrolled loops.
541
542 --------------------------------------------------------------------- */
543