1 /* 2 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved. 4 * Copyright 2015 John Marino <draco@marino.st> 5 * 6 * This source code is derived from the illumos localedef command, and 7 * provided under BSD-style license terms by Nexenta Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * The functions in this file convert from the standard multibyte forms 34 * to the wide character forms used internally by libc. Unfortunately, 35 * this approach means that we need a method for each and every encoding. 36 */ 37 38 #include <stdlib.h> 39 #include <wchar.h> 40 #include <string.h> 41 #include <sys/types.h> 42 #include "localedef.h" 43 44 static int towide_none(wchar_t *, const char *, unsigned); 45 static int towide_utf8(wchar_t *, const char *, unsigned); 46 static int towide_big5(wchar_t *, const char *, unsigned); 47 static int towide_gbk(wchar_t *, const char *, unsigned); 48 static int towide_gb2312(wchar_t *, const char *, unsigned); 49 static int towide_gb18030(wchar_t *, const char *, unsigned); 50 static int towide_mskanji(wchar_t *, const char *, unsigned); 51 static int towide_euccn(wchar_t *, const char *, unsigned); 52 static int towide_eucjp(wchar_t *, const char *, unsigned); 53 static int towide_euckr(wchar_t *, const char *, unsigned); 54 static int towide_euctw(wchar_t *, const char *, unsigned); 55 56 static int tomb_none(char *, wchar_t); 57 static int tomb_utf8(char *, wchar_t); 58 static int tomb_mbs(char *, wchar_t); 59 60 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none; 61 static int (*_tomb)(char *, wchar_t) = tomb_none; 62 static char _encoding_buffer[20] = {'N','O','N','E'}; 63 static const char *_encoding = _encoding_buffer; 64 static int _nbits = 7; 65 66 /* 67 * Table of supported encodings. We only bother to list the multibyte 68 * encodings here, because single byte locales are handed by "NONE". 69 */ 70 static struct { 71 const char *name; 72 /* the name that the underlying libc implemenation uses */ 73 const char *cname; 74 /* the maximum number of bits required for priorities */ 75 int nbits; 76 int (*towide)(wchar_t *, const char *, unsigned); 77 int (*tomb)(char *, wchar_t); 78 } mb_encodings[] = { 79 /* 80 * UTF8 values max out at 0x1fffff (although in theory there could 81 * be later extensions, but it won't happen.) This means we only need 82 * 21 bits to be able to encode the entire range of priorities. 83 */ 84 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 85 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 86 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 87 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, 88 89 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs }, 90 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs }, 91 /* 92 * Becuase the 3-byte form of EUC-JP use the same leading byte, 93 * only 17 bits required to provide unique priorities. (The low 94 * bit of that first byte is set.) By setting this value low, 95 * we can get by with only 3 bytes in the strxfrm expansion. 96 */ 97 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, 98 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, 99 100 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs }, 101 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs }, 102 /* 103 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the 104 * high order byte is 0x8E. However, with 4 byte encodings, 105 * the third byte will be A0-B0. So we only need to consider 106 * the lower order 24 bits for collation. 107 */ 108 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs }, 109 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs }, 110 111 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, 112 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, 113 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs }, 114 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, 115 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, 116 117 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs }, 118 { "big5", "BIG5", 16, towide_big5, tomb_mbs }, 119 { "Big5", "BIG5", 16, towide_big5, tomb_mbs }, 120 121 { "GBK", "GBK", 16, towide_gbk, tomb_mbs }, 122 123 /* 124 * GB18030 can get away with just 31 bits. This is because the 125 * high order bit is always set for 4 byte values, and the 126 * at least one of the other bits in that 4 byte value will 127 * be non-zero. 128 */ 129 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs }, 130 131 /* 132 * This should probably be an aliase for euc-cn, or vice versa. 133 */ 134 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs }, 135 136 { NULL, NULL, 0, 0, 0 }, 137 }; 138 139 static char * 140 show_mb(const char *mb) 141 { 142 static char buf[64]; 143 144 /* ASCII stuff we just print */ 145 if (isascii(*mb) && isgraph(*mb)) { 146 buf[0] = *mb; 147 buf[1] = 0; 148 return (buf); 149 } 150 buf[0] = 0; 151 while (*mb != 0) { 152 char scr[8]; 153 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb); 154 (void) strlcat(buf, scr, sizeof (buf)); 155 mb++; 156 } 157 return (buf); 158 } 159 160 static char *widemsg; 161 162 void 163 werr(const char *fmt, ...) 164 { 165 char *msg; 166 167 va_list va; 168 va_start(va, fmt); 169 (void) vasprintf(&msg, fmt, va); 170 va_end(va); 171 172 free(widemsg); 173 widemsg = msg; 174 } 175 176 /* 177 * This is used for 8-bit encodings. 178 */ 179 int 180 towide_none(wchar_t *c, const char *mb, unsigned n __unused) 181 { 182 if (mb_cur_max != 1) { 183 werr("invalid or unsupported multibyte locale"); 184 return (-1); 185 } 186 *c = (uint8_t)*mb; 187 return (1); 188 } 189 190 int 191 tomb_none(char *mb, wchar_t wc) 192 { 193 if (mb_cur_max != 1) { 194 werr("invalid or unsupported multibyte locale"); 195 return (-1); 196 } 197 *(uint8_t *)mb = (wc & 0xff); 198 mb[1] = 0; 199 return (1); 200 } 201 202 /* 203 * UTF-8 stores wide characters in UTF-32 form. 204 */ 205 int 206 towide_utf8(wchar_t *wc, const char *mb, unsigned n) 207 { 208 wchar_t c; 209 int nb; 210 int lv; /* lowest legal value */ 211 int i; 212 const uint8_t *s = (const uint8_t *)mb; 213 214 c = *s; 215 216 if ((c & 0x80) == 0) { 217 /* 7-bit ASCII */ 218 *wc = c; 219 return (1); 220 } else if ((c & 0xe0) == 0xc0) { 221 /* u80-u7ff - two bytes encoded */ 222 nb = 2; 223 lv = 0x80; 224 c &= ~0xe0; 225 } else if ((c & 0xf0) == 0xe0) { 226 /* u800-uffff - three bytes encoded */ 227 nb = 3; 228 lv = 0x800; 229 c &= ~0xf0; 230 } else if ((c & 0xf8) == 0xf0) { 231 /* u1000-u1fffff - four bytes encoded */ 232 nb = 4; 233 lv = 0x1000; 234 c &= ~0xf8; 235 } else { 236 /* 5 and 6 byte encodings are not legal unicode */ 237 werr("utf8 encoding too large (%s)", show_mb(mb)); 238 return (-1); 239 } 240 if (nb > (int)n) { 241 werr("incomplete utf8 sequence (%s)", show_mb(mb)); 242 return (-1); 243 } 244 245 for (i = 1; i < nb; i++) { 246 if (((s[i]) & 0xc0) != 0x80) { 247 werr("illegal utf8 byte (%x)", s[i]); 248 return (-1); 249 } 250 c <<= 6; 251 c |= (s[i] & 0x3f); 252 } 253 254 if (c < lv) { 255 werr("illegal redundant utf8 encoding (%s)", show_mb(mb)); 256 return (-1); 257 } 258 *wc = c; 259 return (nb); 260 } 261 262 int 263 tomb_utf8(char *mb, wchar_t wc) 264 { 265 uint8_t *s = (uint8_t *)mb; 266 uint8_t msk; 267 int cnt; 268 int i; 269 270 if (wc <= 0x7f) { 271 s[0] = wc & 0x7f; 272 s[1] = 0; 273 return (1); 274 } 275 if (wc <= 0x7ff) { 276 cnt = 2; 277 msk = 0xc0; 278 } else if (wc <= 0xffff) { 279 cnt = 3; 280 msk = 0xe0; 281 } else if (wc <= 0x1fffff) { 282 cnt = 4; 283 msk = 0xf0; 284 } else { 285 werr("illegal uf8 char (%x)", wc); 286 return (-1); 287 } 288 for (i = cnt - 1; i; i--) { 289 s[i] = (wc & 0x3f) | 0x80; 290 wc >>= 6; 291 } 292 s[0] = (msk) | wc; 293 s[cnt] = 0; 294 return (cnt); 295 } 296 297 /* 298 * Several encodings share a simplistic dual byte encoding. In these 299 * forms, they all indicate that a two byte sequence is to be used if 300 * the first byte has its high bit set. They all store this simple 301 * encoding as a 16-bit value, although a great many of the possible 302 * code points are not used in most character sets. This gives a possible 303 * set of just over 32,000 valid code points. 304 * 305 * 0x00 - 0x7f - 1 byte encoding 306 * 0x80 - 0x7fff - illegal 307 * 0x8000 - 0xffff - 2 byte encoding 308 */ 309 310 #pragma GCC diagnostic push 311 #pragma GCC diagnostic ignored "-Wcast-qual" 312 313 static int 314 towide_dbcs(wchar_t *wc, const char *mb, unsigned n) 315 { 316 wchar_t c; 317 318 c = *(uint8_t *)mb; 319 320 if ((c & 0x80) == 0) { 321 /* 7-bit */ 322 *wc = c; 323 return (1); 324 } 325 if (n < 2) { 326 werr("incomplete character sequence (%s)", show_mb(mb)); 327 return (-1); 328 } 329 330 /* Store both bytes as a single 16-bit wide. */ 331 c <<= 8; 332 c |= (uint8_t)(mb[1]); 333 *wc = c; 334 return (2); 335 } 336 337 /* 338 * Most multibyte locales just convert the wide character to the multibyte 339 * form by stripping leading null bytes, and writing the 32-bit quantity 340 * in big-endian order. 341 */ 342 int 343 tomb_mbs(char *mb, wchar_t wc) 344 { 345 uint8_t *s = (uint8_t *)mb; 346 int n = 0, c; 347 348 if ((wc & 0xff000000U) != 0) { 349 n = 4; 350 } else if ((wc & 0x00ff0000U) != 0) { 351 n = 3; 352 } else if ((wc & 0x0000ff00U) != 0) { 353 n = 2; 354 } else { 355 n = 1; 356 } 357 c = n; 358 while (n) { 359 n--; 360 s[n] = wc & 0xff; 361 wc >>= 8; 362 } 363 /* ensure null termination */ 364 s[c] = 0; 365 return (c); 366 } 367 368 369 /* 370 * big5 is a simple dual byte character set. 371 */ 372 int 373 towide_big5(wchar_t *wc, const char *mb, unsigned n) 374 { 375 return (towide_dbcs(wc, mb, n)); 376 } 377 378 /* 379 * GBK encodes wides in the same way that big5 does, the high order 380 * bit of the first byte indicates a double byte character. 381 */ 382 int 383 towide_gbk(wchar_t *wc, const char *mb, unsigned n) 384 { 385 return (towide_dbcs(wc, mb, n)); 386 } 387 388 /* 389 * GB2312 is another DBCS. Its cleaner than others in that the second 390 * byte does not encode ASCII, but it supports characters. 391 */ 392 int 393 towide_gb2312(wchar_t *wc, const char *mb, unsigned n) 394 { 395 return (towide_dbcs(wc, mb, n)); 396 } 397 398 /* 399 * GB18030. This encodes as 8, 16, or 32-bits. 400 * 7-bit values are in 1 byte, 4 byte sequences are used when 401 * the second byte encodes 0x30-39 and all other sequences are 2 bytes. 402 */ 403 int 404 towide_gb18030(wchar_t *wc, const char *mb, unsigned n) 405 { 406 wchar_t c; 407 408 c = *(uint8_t *)mb; 409 410 if ((c & 0x80) == 0) { 411 /* 7-bit */ 412 *wc = c; 413 return (1); 414 } 415 if (n < 2) { 416 werr("incomplete character sequence (%s)", show_mb(mb)); 417 return (-1); 418 } 419 420 /* pull in the second byte */ 421 c <<= 8; 422 c |= (uint8_t)(mb[1]); 423 424 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) { 425 if (n < 4) { 426 werr("incomplete 4-byte character sequence (%s)", 427 show_mb(mb)); 428 return (-1); 429 } 430 c <<= 8; 431 c |= (uint8_t)(mb[2]); 432 c <<= 8; 433 c |= (uint8_t)(mb[3]); 434 *wc = c; 435 return (4); 436 } 437 438 *wc = c; 439 return (2); 440 } 441 442 /* 443 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it 444 * also has a range of single byte characters above 0x80. (0xa1-0xdf). 445 */ 446 int 447 towide_mskanji(wchar_t *wc, const char *mb, unsigned n) 448 { 449 wchar_t c; 450 451 c = *(uint8_t *)mb; 452 453 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) { 454 /* 7-bit */ 455 *wc = c; 456 return (1); 457 } 458 459 if (n < 2) { 460 werr("incomplete character sequence (%s)", show_mb(mb)); 461 return (-1); 462 } 463 464 /* Store both bytes as a single 16-bit wide. */ 465 c <<= 8; 466 c |= (uint8_t)(mb[1]); 467 *wc = c; 468 return (2); 469 } 470 471 /* 472 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional 473 * variable data to encode these, but we're going to treat each as independent 474 * instead. Its the only way we can sensibly move forward. 475 * 476 * Note that the way in which the different EUC forms vary is how wide 477 * CS2 and CS3 are and what the first byte of them is. 478 */ 479 static int 480 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n, 481 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width) 482 { 483 int i; 484 int width = 2; 485 wchar_t c; 486 487 c = *(uint8_t *)mb; 488 489 /* 490 * All variations of EUC encode 7-bit ASCII as one byte, and use 491 * additional bytes for more than that. 492 */ 493 if ((c & 0x80) == 0) { 494 /* 7-bit */ 495 *wc = c; 496 return (1); 497 } 498 499 /* 500 * All EUC variants reserve 0xa1-0xff to identify CS1, which 501 * is always two bytes wide. Note that unused CS will be zero, 502 * and that cannot be true because we know that the high order 503 * bit must be set. 504 */ 505 if (c >= 0xa1) { 506 width = 2; 507 } else if (c == cs2) { 508 width = cs2width; 509 } else if (c == cs3) { 510 width = cs3width; 511 } 512 513 if ((int)n < width) { 514 werr("incomplete character sequence (%s)", show_mb(mb)); 515 return (-1); 516 } 517 518 for (i = 1; i < width; i++) { 519 /* pull in the next byte */ 520 c <<= 8; 521 c |= (uint8_t)(mb[i]); 522 } 523 524 *wc = c; 525 return (width); 526 } 527 528 #pragma GCC diagnostic pop 529 530 /* 531 * EUC-CN encodes as follows: 532 * 533 * Code set 0 (ASCII): 0x21-0x7E 534 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE 535 * Code set 2: unused 536 * Code set 3: unused 537 */ 538 int 539 towide_euccn(wchar_t *wc, const char *mb, unsigned n) 540 { 541 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); 542 } 543 544 /* 545 * EUC-JP encodes as follows: 546 * 547 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E 548 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE 549 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF 550 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE 551 */ 552 int 553 towide_eucjp(wchar_t *wc, const char *mb, unsigned n) 554 { 555 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3)); 556 } 557 558 /* 559 * EUC-KR encodes as follows: 560 * 561 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E 562 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE 563 * Code set 2: unused 564 * Code set 3: unused 565 */ 566 int 567 towide_euckr(wchar_t *wc, const char *mb, unsigned n) 568 { 569 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0)); 570 } 571 572 /* 573 * EUC-TW encodes as follows: 574 * 575 * Code set 0 (ASCII): 0x21-0x7E 576 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE 577 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE 578 * Code set 3: unused 579 */ 580 int 581 towide_euctw(wchar_t *wc, const char *mb, unsigned n) 582 { 583 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); 584 } 585 586 /* 587 * Public entry points. 588 */ 589 590 int 591 to_wide(wchar_t *wc, const char *mb) 592 { 593 /* this won't fail hard */ 594 return (_towide(wc, mb, strlen(mb))); 595 } 596 597 int 598 to_mb(char *mb, wchar_t wc) 599 { 600 int rv; 601 602 if ((rv = _tomb(mb, wc)) < 0) { 603 errf(widemsg); 604 free(widemsg); 605 widemsg = NULL; 606 } 607 return (rv); 608 } 609 610 char * 611 to_mb_string(const wchar_t *wcs) 612 { 613 char *mbs; 614 char *ptr; 615 int len; 616 617 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1); 618 if (mbs == NULL) { 619 errf("out of memory"); 620 return (NULL); 621 } 622 ptr = mbs; 623 while (*wcs) { 624 if ((len = to_mb(ptr, *wcs)) < 0) { 625 INTERR; 626 free(mbs); 627 return (NULL); 628 } 629 wcs++; 630 ptr += len; 631 } 632 *ptr = 0; 633 return (mbs); 634 } 635 636 void 637 set_wide_encoding(const char *encoding) 638 { 639 int i; 640 641 _towide = towide_none; 642 _tomb = tomb_none; 643 _nbits = 8; 644 645 strcpy (_encoding_buffer, "NONE:"); 646 strncat (_encoding_buffer, encoding, 14); 647 for (i = 0; mb_encodings[i].name; i++) { 648 if (strcasecmp(encoding, mb_encodings[i].name) == 0) { 649 _towide = mb_encodings[i].towide; 650 _tomb = mb_encodings[i].tomb; 651 _encoding = mb_encodings[i].cname; 652 _nbits = mb_encodings[i].nbits; 653 break; 654 } 655 } 656 } 657 658 const char * 659 get_wide_encoding(void) 660 { 661 return (_encoding); 662 } 663 664 int 665 max_wide(void) 666 { 667 return ((int)((1U << _nbits) - 1)); 668 } 669