1 /*===--- ConvertUTF.c - Universal Character Names conversions ---------------=== 2 * 3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 * See https://llvm.org/LICENSE.txt for license information. 5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 * 7 *===------------------------------------------------------------------------=*/ 8 /* 9 * Copyright 2001-2004 Unicode, Inc. 10 * 11 * Disclaimer 12 * 13 * This source code is provided as is by Unicode, Inc. No claims are 14 * made as to fitness for any particular purpose. No warranties of any 15 * kind are expressed or implied. The recipient agrees to determine 16 * applicability of information provided. If this file has been 17 * purchased on magnetic or optical media from Unicode, Inc., the 18 * sole remedy for any claim will be exchange of defective media 19 * within 90 days of receipt. 20 * 21 * Limitations on Rights to Redistribute This Code 22 * 23 * Unicode, Inc. hereby grants the right to freely use the information 24 * supplied in this file in the creation of products supporting the 25 * Unicode Standard, and to make copies of this file in any form 26 * for internal or external distribution as long as this notice 27 * remains attached. 28 */ 29 30 /* --------------------------------------------------------------------- 31 32 Conversions between UTF32, UTF-16, and UTF-8. Source code file. 33 Author: Mark E. Davis, 1994. 34 Rev History: Rick McGowan, fixes & updates May 2001. 35 Sept 2001: fixed const & error conditions per 36 mods suggested by S. Parent & A. Lillich. 37 June 2002: Tim Dodd added detection and handling of incomplete 38 source sequences, enhanced error detection, added casts 39 to eliminate compiler warnings. 40 July 2003: slight mods to back out aggressive FFFE detection. 41 Jan 2004: updated switches in from-UTF8 conversions. 42 Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. 43 44 See the header file "ConvertUTF.h" for complete documentation. 45 46 ------------------------------------------------------------------------ */ 47 48 #include "llvm/Support/ConvertUTF.h" 49 #ifdef CVTUTF_DEBUG 50 #include <stdio.h> 51 #endif 52 #include <assert.h> 53 54 /* 55 * This code extensively uses fall-through switches. 56 * Keep the compiler from warning about that. 57 */ 58 #if defined(__clang__) && defined(__has_warning) 59 # if __has_warning("-Wimplicit-fallthrough") 60 # define ConvertUTF_DISABLE_WARNINGS \ 61 _Pragma("clang diagnostic push") \ 62 _Pragma("clang diagnostic ignored \"-Wimplicit-fallthrough\"") 63 # define ConvertUTF_RESTORE_WARNINGS \ 64 _Pragma("clang diagnostic pop") 65 # endif 66 #elif defined(__GNUC__) && __GNUC__ > 6 67 # define ConvertUTF_DISABLE_WARNINGS \ 68 _Pragma("GCC diagnostic push") \ 69 _Pragma("GCC diagnostic ignored \"-Wimplicit-fallthrough\"") 70 # define ConvertUTF_RESTORE_WARNINGS \ 71 _Pragma("GCC diagnostic pop") 72 #endif 73 #ifndef ConvertUTF_DISABLE_WARNINGS 74 # define ConvertUTF_DISABLE_WARNINGS 75 #endif 76 #ifndef ConvertUTF_RESTORE_WARNINGS 77 # define ConvertUTF_RESTORE_WARNINGS 78 #endif 79 80 ConvertUTF_DISABLE_WARNINGS 81 82 namespace llvm { 83 84 static const int halfShift = 10; /* used for shifting by 10 bits */ 85 86 static const UTF32 halfBase = 0x0010000UL; 87 static const UTF32 halfMask = 0x3FFUL; 88 89 #define UNI_SUR_HIGH_START (UTF32)0xD800 90 #define UNI_SUR_HIGH_END (UTF32)0xDBFF 91 #define UNI_SUR_LOW_START (UTF32)0xDC00 92 #define UNI_SUR_LOW_END (UTF32)0xDFFF 93 94 /* --------------------------------------------------------------------- */ 95 96 /* 97 * Index into the table below with the first byte of a UTF-8 sequence to 98 * get the number of trailing bytes that are supposed to follow it. 99 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is 100 * left as-is for anyone who may want to do such conversion, which was 101 * allowed in earlier algorithms. 102 */ 103 static const char trailingBytesForUTF8[256] = { 104 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 105 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 106 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 107 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 108 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 109 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 110 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 111 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 112 }; 113 114 /* 115 * Magic values subtracted from a buffer value during UTF8 conversion. 116 * This table contains as many values as there might be trailing bytes 117 * in a UTF-8 sequence. 118 */ 119 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 120 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; 121 122 /* 123 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed 124 * into the first byte, depending on how many bytes follow. There are 125 * as many entries in this table as there are UTF-8 sequence types. 126 * (I.e., one byte sequence, two byte... etc.). Remember that sequencs 127 * for *legal* UTF-8 will be 4 or fewer bytes total. 128 */ 129 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; 130 131 /* --------------------------------------------------------------------- */ 132 133 /* The interface converts a whole buffer to avoid function-call overhead. 134 * Constants have been gathered. Loops & conditionals have been removed as 135 * much as possible for efficiency, in favor of drop-through switches. 136 * (See "Note A" at the bottom of the file for equivalent code.) 137 * If your compiler supports it, the "isLegalUTF8" call can be turned 138 * into an inline function. 139 */ 140 141 142 /* --------------------------------------------------------------------- */ 143 144 ConversionResult ConvertUTF32toUTF16 ( 145 const UTF32** sourceStart, const UTF32* sourceEnd, 146 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { 147 ConversionResult result = conversionOK; 148 const UTF32* source = *sourceStart; 149 UTF16* target = *targetStart; 150 while (source < sourceEnd) { 151 UTF32 ch; 152 if (target >= targetEnd) { 153 result = targetExhausted; break; 154 } 155 ch = *source++; 156 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ 157 /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ 158 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 159 if (flags == strictConversion) { 160 --source; /* return to the illegal value itself */ 161 result = sourceIllegal; 162 break; 163 } else { 164 *target++ = UNI_REPLACEMENT_CHAR; 165 } 166 } else { 167 *target++ = (UTF16)ch; /* normal case */ 168 } 169 } else if (ch > UNI_MAX_LEGAL_UTF32) { 170 if (flags == strictConversion) { 171 result = sourceIllegal; 172 } else { 173 *target++ = UNI_REPLACEMENT_CHAR; 174 } 175 } else { 176 /* target is a character in range 0xFFFF - 0x10FFFF. */ 177 if (target + 1 >= targetEnd) { 178 --source; /* Back up source pointer! */ 179 result = targetExhausted; break; 180 } 181 ch -= halfBase; 182 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); 183 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); 184 } 185 } 186 *sourceStart = source; 187 *targetStart = target; 188 return result; 189 } 190 191 /* --------------------------------------------------------------------- */ 192 193 ConversionResult ConvertUTF16toUTF32 ( 194 const UTF16** sourceStart, const UTF16* sourceEnd, 195 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { 196 ConversionResult result = conversionOK; 197 const UTF16* source = *sourceStart; 198 UTF32* target = *targetStart; 199 UTF32 ch, ch2; 200 while (source < sourceEnd) { 201 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ 202 ch = *source++; 203 /* If we have a surrogate pair, convert to UTF32 first. */ 204 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { 205 /* If the 16 bits following the high surrogate are in the source buffer... */ 206 if (source < sourceEnd) { 207 ch2 = *source; 208 /* If it's a low surrogate, convert to UTF32. */ 209 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { 210 ch = ((ch - UNI_SUR_HIGH_START) << halfShift) 211 + (ch2 - UNI_SUR_LOW_START) + halfBase; 212 ++source; 213 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ 214 --source; /* return to the illegal value itself */ 215 result = sourceIllegal; 216 break; 217 } 218 } else { /* We don't have the 16 bits following the high surrogate. */ 219 --source; /* return to the high surrogate */ 220 result = sourceExhausted; 221 break; 222 } 223 } else if (flags == strictConversion) { 224 /* UTF-16 surrogate values are illegal in UTF-32 */ 225 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { 226 --source; /* return to the illegal value itself */ 227 result = sourceIllegal; 228 break; 229 } 230 } 231 if (target >= targetEnd) { 232 source = oldSource; /* Back up source pointer! */ 233 result = targetExhausted; break; 234 } 235 *target++ = ch; 236 } 237 *sourceStart = source; 238 *targetStart = target; 239 #ifdef CVTUTF_DEBUG 240 if (result == sourceIllegal) { 241 fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); 242 fflush(stderr); 243 } 244 #endif 245 return result; 246 } 247 ConversionResult ConvertUTF16toUTF8 ( 248 const UTF16** sourceStart, const UTF16* sourceEnd, 249 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { 250 ConversionResult result = conversionOK; 251 const UTF16* source = *sourceStart; 252 UTF8* target = *targetStart; 253 while (source < sourceEnd) { 254 UTF32 ch; 255 unsigned short bytesToWrite = 0; 256 const UTF32 byteMask = 0xBF; 257 const UTF32 byteMark = 0x80; 258 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ 259 ch = *source++; 260 /* If we have a surrogate pair, convert to UTF32 first. */ 261 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { 262 /* If the 16 bits following the high surrogate are in the source buffer... */ 263 if (source < sourceEnd) { 264 UTF32 ch2 = *source; 265 /* If it's a low surrogate, convert to UTF32. */ 266 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { 267 ch = ((ch - UNI_SUR_HIGH_START) << halfShift) 268 + (ch2 - UNI_SUR_LOW_START) + halfBase; 269 ++source; 270 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ 271 --source; /* return to the illegal value itself */ 272 result = sourceIllegal; 273 break; 274 } 275 } else { /* We don't have the 16 bits following the high surrogate. */ 276 --source; /* return to the high surrogate */ 277 result = sourceExhausted; 278 break; 279 } 280 } else if (flags == strictConversion) { 281 /* UTF-16 surrogate values are illegal in UTF-32 */ 282 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { 283 --source; /* return to the illegal value itself */ 284 result = sourceIllegal; 285 break; 286 } 287 } 288 /* Figure out how many bytes the result will require */ 289 if (ch < (UTF32)0x80) { bytesToWrite = 1; 290 } else if (ch < (UTF32)0x800) { bytesToWrite = 2; 291 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; 292 } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; 293 } else { bytesToWrite = 3; 294 ch = UNI_REPLACEMENT_CHAR; 295 } 296 297 target += bytesToWrite; 298 if (target > targetEnd) { 299 source = oldSource; /* Back up source pointer! */ 300 target -= bytesToWrite; result = targetExhausted; break; 301 } 302 switch (bytesToWrite) { /* note: everything falls through. */ 303 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 304 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 305 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 306 case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); 307 } 308 target += bytesToWrite; 309 } 310 *sourceStart = source; 311 *targetStart = target; 312 return result; 313 } 314 315 /* --------------------------------------------------------------------- */ 316 317 ConversionResult ConvertUTF32toUTF8 ( 318 const UTF32** sourceStart, const UTF32* sourceEnd, 319 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { 320 ConversionResult result = conversionOK; 321 const UTF32* source = *sourceStart; 322 UTF8* target = *targetStart; 323 while (source < sourceEnd) { 324 UTF32 ch; 325 unsigned short bytesToWrite = 0; 326 const UTF32 byteMask = 0xBF; 327 const UTF32 byteMark = 0x80; 328 ch = *source++; 329 if (flags == strictConversion ) { 330 /* UTF-16 surrogate values are illegal in UTF-32 */ 331 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 332 --source; /* return to the illegal value itself */ 333 result = sourceIllegal; 334 break; 335 } 336 } 337 /* 338 * Figure out how many bytes the result will require. Turn any 339 * illegally large UTF32 things (> Plane 17) into replacement chars. 340 */ 341 if (ch < (UTF32)0x80) { bytesToWrite = 1; 342 } else if (ch < (UTF32)0x800) { bytesToWrite = 2; 343 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; 344 } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; 345 } else { bytesToWrite = 3; 346 ch = UNI_REPLACEMENT_CHAR; 347 result = sourceIllegal; 348 } 349 350 target += bytesToWrite; 351 if (target > targetEnd) { 352 --source; /* Back up source pointer! */ 353 target -= bytesToWrite; result = targetExhausted; break; 354 } 355 switch (bytesToWrite) { /* note: everything falls through. */ 356 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 357 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 358 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 359 case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); 360 } 361 target += bytesToWrite; 362 } 363 *sourceStart = source; 364 *targetStart = target; 365 return result; 366 } 367 368 /* --------------------------------------------------------------------- */ 369 370 /* 371 * Utility routine to tell whether a sequence of bytes is legal UTF-8. 372 * This must be called with the length pre-determined by the first byte. 373 * If not calling this from ConvertUTF8to*, then the length can be set by: 374 * length = trailingBytesForUTF8[*source]+1; 375 * and the sequence is illegal right away if there aren't that many bytes 376 * available. 377 * If presented with a length > 4, this returns false. The Unicode 378 * definition of UTF-8 goes up to 4-byte sequences. 379 */ 380 381 static Boolean isLegalUTF8(const UTF8 *source, int length) { 382 UTF8 a; 383 const UTF8 *srcptr = source+length; 384 switch (length) { 385 default: return false; 386 /* Everything else falls through when "true"... */ 387 case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 388 case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 389 case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 390 391 switch (*source) { 392 /* no fall-through in this inner switch */ 393 case 0xE0: if (a < 0xA0) return false; break; 394 case 0xED: if (a > 0x9F) return false; break; 395 case 0xF0: if (a < 0x90) return false; break; 396 case 0xF4: if (a > 0x8F) return false; break; 397 default: if (a < 0x80) return false; 398 } 399 400 case 1: if (*source >= 0x80 && *source < 0xC2) return false; 401 } 402 if (*source > 0xF4) return false; 403 return true; 404 } 405 406 /* --------------------------------------------------------------------- */ 407 408 /* 409 * Exported function to return whether a UTF-8 sequence is legal or not. 410 * This is not used here; it's just exported. 411 */ 412 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { 413 int length = trailingBytesForUTF8[*source]+1; 414 if (length > sourceEnd - source) { 415 return false; 416 } 417 return isLegalUTF8(source, length); 418 } 419 420 /* 421 * Exported function to return the size of the first utf-8 code unit sequence, 422 * Or 0 if the sequence is not valid; 423 */ 424 unsigned getUTF8SequenceSize(const UTF8 *source, const UTF8 *sourceEnd) { 425 int length = trailingBytesForUTF8[*source] + 1; 426 return (length <= sourceEnd - source && isLegalUTF8(source, length)) ? length 427 : 0; 428 } 429 430 /* --------------------------------------------------------------------- */ 431 432 static unsigned 433 findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source, 434 const UTF8 *sourceEnd) { 435 UTF8 b1, b2, b3; 436 437 assert(!isLegalUTF8Sequence(source, sourceEnd)); 438 439 /* 440 * Unicode 6.3.0, D93b: 441 * 442 * Maximal subpart of an ill-formed subsequence: The longest code unit 443 * subsequence starting at an unconvertible offset that is either: 444 * a. the initial subsequence of a well-formed code unit sequence, or 445 * b. a subsequence of length one. 446 */ 447 448 if (source == sourceEnd) 449 return 0; 450 451 /* 452 * Perform case analysis. See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8 453 * Byte Sequences. 454 */ 455 456 b1 = *source; 457 ++source; 458 if (b1 >= 0xC2 && b1 <= 0xDF) { 459 /* 460 * First byte is valid, but we know that this code unit sequence is 461 * invalid, so the maximal subpart has to end after the first byte. 462 */ 463 return 1; 464 } 465 466 if (source == sourceEnd) 467 return 1; 468 469 b2 = *source; 470 ++source; 471 472 if (b1 == 0xE0) { 473 return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1; 474 } 475 if (b1 >= 0xE1 && b1 <= 0xEC) { 476 return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1; 477 } 478 if (b1 == 0xED) { 479 return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1; 480 } 481 if (b1 >= 0xEE && b1 <= 0xEF) { 482 return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1; 483 } 484 if (b1 == 0xF0) { 485 if (b2 >= 0x90 && b2 <= 0xBF) { 486 if (source == sourceEnd) 487 return 2; 488 489 b3 = *source; 490 return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; 491 } 492 return 1; 493 } 494 if (b1 >= 0xF1 && b1 <= 0xF3) { 495 if (b2 >= 0x80 && b2 <= 0xBF) { 496 if (source == sourceEnd) 497 return 2; 498 499 b3 = *source; 500 return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; 501 } 502 return 1; 503 } 504 if (b1 == 0xF4) { 505 if (b2 >= 0x80 && b2 <= 0x8F) { 506 if (source == sourceEnd) 507 return 2; 508 509 b3 = *source; 510 return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; 511 } 512 return 1; 513 } 514 515 assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5); 516 /* 517 * There are no valid sequences that start with these bytes. Maximal subpart 518 * is defined to have length 1 in these cases. 519 */ 520 return 1; 521 } 522 523 /* --------------------------------------------------------------------- */ 524 525 /* 526 * Exported function to return the total number of bytes in a codepoint 527 * represented in UTF-8, given the value of the first byte. 528 */ 529 unsigned getNumBytesForUTF8(UTF8 first) { 530 return trailingBytesForUTF8[first] + 1; 531 } 532 533 /* --------------------------------------------------------------------- */ 534 535 /* 536 * Exported function to return whether a UTF-8 string is legal or not. 537 * This is not used here; it's just exported. 538 */ 539 Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) { 540 while (*source != sourceEnd) { 541 int length = trailingBytesForUTF8[**source] + 1; 542 if (length > sourceEnd - *source || !isLegalUTF8(*source, length)) 543 return false; 544 *source += length; 545 } 546 return true; 547 } 548 549 /* --------------------------------------------------------------------- */ 550 551 ConversionResult ConvertUTF8toUTF16 ( 552 const UTF8** sourceStart, const UTF8* sourceEnd, 553 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { 554 ConversionResult result = conversionOK; 555 const UTF8* source = *sourceStart; 556 UTF16* target = *targetStart; 557 while (source < sourceEnd) { 558 UTF32 ch = 0; 559 unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; 560 if (extraBytesToRead >= sourceEnd - source) { 561 result = sourceExhausted; break; 562 } 563 /* Do this check whether lenient or strict */ 564 if (!isLegalUTF8(source, extraBytesToRead+1)) { 565 result = sourceIllegal; 566 break; 567 } 568 /* 569 * The cases all fall through. See "Note A" below. 570 */ 571 switch (extraBytesToRead) { 572 case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ 573 case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ 574 case 3: ch += *source++; ch <<= 6; 575 case 2: ch += *source++; ch <<= 6; 576 case 1: ch += *source++; ch <<= 6; 577 case 0: ch += *source++; 578 } 579 ch -= offsetsFromUTF8[extraBytesToRead]; 580 581 if (target >= targetEnd) { 582 source -= (extraBytesToRead+1); /* Back up source pointer! */ 583 result = targetExhausted; break; 584 } 585 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ 586 /* UTF-16 surrogate values are illegal in UTF-32 */ 587 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 588 if (flags == strictConversion) { 589 source -= (extraBytesToRead+1); /* return to the illegal value itself */ 590 result = sourceIllegal; 591 break; 592 } else { 593 *target++ = UNI_REPLACEMENT_CHAR; 594 } 595 } else { 596 *target++ = (UTF16)ch; /* normal case */ 597 } 598 } else if (ch > UNI_MAX_UTF16) { 599 if (flags == strictConversion) { 600 result = sourceIllegal; 601 source -= (extraBytesToRead+1); /* return to the start */ 602 break; /* Bail out; shouldn't continue */ 603 } else { 604 *target++ = UNI_REPLACEMENT_CHAR; 605 } 606 } else { 607 /* target is a character in range 0xFFFF - 0x10FFFF. */ 608 if (target + 1 >= targetEnd) { 609 source -= (extraBytesToRead+1); /* Back up source pointer! */ 610 result = targetExhausted; break; 611 } 612 ch -= halfBase; 613 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); 614 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); 615 } 616 } 617 *sourceStart = source; 618 *targetStart = target; 619 return result; 620 } 621 622 /* --------------------------------------------------------------------- */ 623 624 static ConversionResult ConvertUTF8toUTF32Impl( 625 const UTF8** sourceStart, const UTF8* sourceEnd, 626 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags, 627 Boolean InputIsPartial) { 628 ConversionResult result = conversionOK; 629 const UTF8* source = *sourceStart; 630 UTF32* target = *targetStart; 631 while (source < sourceEnd) { 632 UTF32 ch = 0; 633 unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; 634 if (extraBytesToRead >= sourceEnd - source) { 635 if (flags == strictConversion || InputIsPartial) { 636 result = sourceExhausted; 637 break; 638 } else { 639 result = sourceIllegal; 640 641 /* 642 * Replace the maximal subpart of ill-formed sequence with 643 * replacement character. 644 */ 645 source += findMaximalSubpartOfIllFormedUTF8Sequence(source, 646 sourceEnd); 647 *target++ = UNI_REPLACEMENT_CHAR; 648 continue; 649 } 650 } 651 if (target >= targetEnd) { 652 result = targetExhausted; break; 653 } 654 655 /* Do this check whether lenient or strict */ 656 if (!isLegalUTF8(source, extraBytesToRead+1)) { 657 result = sourceIllegal; 658 if (flags == strictConversion) { 659 /* Abort conversion. */ 660 break; 661 } else { 662 /* 663 * Replace the maximal subpart of ill-formed sequence with 664 * replacement character. 665 */ 666 source += findMaximalSubpartOfIllFormedUTF8Sequence(source, 667 sourceEnd); 668 *target++ = UNI_REPLACEMENT_CHAR; 669 continue; 670 } 671 } 672 /* 673 * The cases all fall through. See "Note A" below. 674 */ 675 switch (extraBytesToRead) { 676 case 5: ch += *source++; ch <<= 6; 677 case 4: ch += *source++; ch <<= 6; 678 case 3: ch += *source++; ch <<= 6; 679 case 2: ch += *source++; ch <<= 6; 680 case 1: ch += *source++; ch <<= 6; 681 case 0: ch += *source++; 682 } 683 ch -= offsetsFromUTF8[extraBytesToRead]; 684 685 if (ch <= UNI_MAX_LEGAL_UTF32) { 686 /* 687 * UTF-16 surrogate values are illegal in UTF-32, and anything 688 * over Plane 17 (> 0x10FFFF) is illegal. 689 */ 690 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 691 if (flags == strictConversion) { 692 source -= (extraBytesToRead+1); /* return to the illegal value itself */ 693 result = sourceIllegal; 694 break; 695 } else { 696 *target++ = UNI_REPLACEMENT_CHAR; 697 } 698 } else { 699 *target++ = ch; 700 } 701 } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ 702 result = sourceIllegal; 703 *target++ = UNI_REPLACEMENT_CHAR; 704 } 705 } 706 *sourceStart = source; 707 *targetStart = target; 708 return result; 709 } 710 711 ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart, 712 const UTF8 *sourceEnd, 713 UTF32 **targetStart, 714 UTF32 *targetEnd, 715 ConversionFlags flags) { 716 return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd, 717 flags, /*InputIsPartial=*/true); 718 } 719 720 ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart, 721 const UTF8 *sourceEnd, UTF32 **targetStart, 722 UTF32 *targetEnd, ConversionFlags flags) { 723 return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd, 724 flags, /*InputIsPartial=*/false); 725 } 726 727 /* --------------------------------------------------------------------- 728 729 Note A. 730 The fall-through switches in UTF-8 reading code save a 731 temp variable, some decrements & conditionals. The switches 732 are equivalent to the following loop: 733 { 734 int tmpBytesToRead = extraBytesToRead+1; 735 do { 736 ch += *source++; 737 --tmpBytesToRead; 738 if (tmpBytesToRead) ch <<= 6; 739 } while (tmpBytesToRead > 0); 740 } 741 In UTF-8 writing code, the switches on "bytesToWrite" are 742 similarly unrolled loops. 743 744 --------------------------------------------------------------------- */ 745 746 } // namespace llvm 747 748 ConvertUTF_RESTORE_WARNINGS 749