1 /*- 2 * Copyright (c) 2003-2011 Tim Kientzle 3 * Copyright (c) 2011-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 __FBSDID("$FreeBSD: head/lib/libarchive/archive_string.c 201095 2009-12-28 02:33:22Z kientzle $"); 29 30 /* 31 * Basic resizable string support, to simplify manipulating arbitrary-sized 32 * strings while minimizing heap activity. 33 * 34 * In particular, the buffer used by a string object is only grown, it 35 * never shrinks, so you can clear and reuse the same string object 36 * without incurring additional memory allocations. 37 */ 38 39 #ifdef HAVE_ERRNO_H 40 #include <errno.h> 41 #endif 42 #ifdef HAVE_ICONV_H 43 #include <iconv.h> 44 #endif 45 #ifdef HAVE_LANGINFO_H 46 #include <langinfo.h> 47 #endif 48 #ifdef HAVE_LOCALCHARSET_H 49 #include <localcharset.h> 50 #endif 51 #ifdef HAVE_STDLIB_H 52 #include <stdlib.h> 53 #endif 54 #ifdef HAVE_STRING_H 55 #include <string.h> 56 #endif 57 #ifdef HAVE_WCHAR_H 58 #include <wchar.h> 59 #endif 60 #if defined(_WIN32) && !defined(__CYGWIN__) 61 #include <windows.h> 62 #include <locale.h> 63 #endif 64 65 #include "archive_endian.h" 66 #include "archive_private.h" 67 #include "archive_string.h" 68 #include "archive_string_composition.h" 69 70 #if !defined(HAVE_WMEMCPY) && !defined(wmemcpy) 71 #define wmemcpy(a,b,i) (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t)) 72 #endif 73 74 #if !defined(HAVE_WMEMMOVE) && !defined(wmemmove) 75 #define wmemmove(a,b,i) (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t)) 76 #endif 77 78 struct archive_string_conv { 79 struct archive_string_conv *next; 80 char *from_charset; 81 char *to_charset; 82 unsigned from_cp; 83 unsigned to_cp; 84 /* Set 1 if from_charset and to_charset are the same. */ 85 int same; 86 int flag; 87 #define SCONV_TO_CHARSET 1 /* MBS is being converted to specified 88 * charset. */ 89 #define SCONV_FROM_CHARSET (1<<1) /* MBS is being converted from 90 * specified charset. */ 91 #define SCONV_BEST_EFFORT (1<<2) /* Copy at least ASCII code. */ 92 #define SCONV_WIN_CP (1<<3) /* Use Windows API for converting 93 * MBS. */ 94 #define SCONV_UTF8_LIBARCHIVE_2 (1<<4) /* Incorrect UTF-8 made by libarchive 95 * 2.x in the wrong assumption. */ 96 #define SCONV_NORMALIZATION_C (1<<6) /* Need normalization to be Form C. 97 * Before UTF-8 characters are actually 98 * processed. */ 99 #define SCONV_NORMALIZATION_D (1<<7) /* Need normalization to be Form D. 100 * Before UTF-8 characters are actually 101 * processed. 102 * Currently this only for MAC OS X. */ 103 #define SCONV_TO_UTF8 (1<<8) /* "to charset" side is UTF-8. */ 104 #define SCONV_FROM_UTF8 (1<<9) /* "from charset" side is UTF-8. */ 105 #define SCONV_TO_UTF16BE (1<<10) /* "to charset" side is UTF-16BE. */ 106 #define SCONV_FROM_UTF16BE (1<<11) /* "from charset" side is UTF-16BE. */ 107 #define SCONV_TO_UTF16LE (1<<12) /* "to charset" side is UTF-16LE. */ 108 #define SCONV_FROM_UTF16LE (1<<13) /* "from charset" side is UTF-16LE. */ 109 #define SCONV_TO_UTF16 (SCONV_TO_UTF16BE | SCONV_TO_UTF16LE) 110 #define SCONV_FROM_UTF16 (SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE) 111 112 #if HAVE_ICONV 113 iconv_t cd; 114 iconv_t cd_w;/* Use at archive_mstring on 115 * Windows. */ 116 #endif 117 /* A temporary buffer for normalization. */ 118 struct archive_string utftmp; 119 int (*converter[2])(struct archive_string *, const void *, size_t, 120 struct archive_string_conv *); 121 int nconverter; 122 }; 123 124 #define CP_C_LOCALE 0 /* "C" locale only for this file. */ 125 #define CP_UTF16LE 1200 126 #define CP_UTF16BE 1201 127 128 #define IS_HIGH_SURROGATE_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDBFF) 129 #define IS_LOW_SURROGATE_LA(uc) ((uc) >= 0xDC00 && (uc) <= 0xDFFF) 130 #define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF) 131 #define UNICODE_MAX 0x10FFFF 132 #define UNICODE_R_CHAR 0xFFFD /* Replacement character. */ 133 /* Set U+FFFD(Replacement character) in UTF-8. */ 134 static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd}; 135 136 static struct archive_string_conv *find_sconv_object(struct archive *, 137 const char *, const char *); 138 static void add_sconv_object(struct archive *, struct archive_string_conv *); 139 static struct archive_string_conv *create_sconv_object(const char *, 140 const char *, unsigned, int); 141 static void free_sconv_object(struct archive_string_conv *); 142 static struct archive_string_conv *get_sconv_object(struct archive *, 143 const char *, const char *, int); 144 static unsigned make_codepage_from_charset(const char *); 145 static unsigned get_current_codepage(void); 146 static unsigned get_current_oemcp(void); 147 static size_t mbsnbytes(const void *, size_t); 148 static size_t utf16nbytes(const void *, size_t); 149 #if defined(_WIN32) && !defined(__CYGWIN__) 150 static int archive_wstring_append_from_mbs_in_codepage( 151 struct archive_wstring *, const char *, size_t, 152 struct archive_string_conv *); 153 static int archive_string_append_from_wcs_in_codepage(struct archive_string *, 154 const wchar_t *, size_t, struct archive_string_conv *); 155 static int is_big_endian(void); 156 static int strncat_in_codepage(struct archive_string *, const void *, 157 size_t, struct archive_string_conv *); 158 static int win_strncat_from_utf16be(struct archive_string *, const void *, 159 size_t, struct archive_string_conv *); 160 static int win_strncat_from_utf16le(struct archive_string *, const void *, 161 size_t, struct archive_string_conv *); 162 static int win_strncat_to_utf16be(struct archive_string *, const void *, 163 size_t, struct archive_string_conv *); 164 static int win_strncat_to_utf16le(struct archive_string *, const void *, 165 size_t, struct archive_string_conv *); 166 #endif 167 static int best_effort_strncat_from_utf16be(struct archive_string *, 168 const void *, size_t, struct archive_string_conv *); 169 static int best_effort_strncat_from_utf16le(struct archive_string *, 170 const void *, size_t, struct archive_string_conv *); 171 static int best_effort_strncat_to_utf16be(struct archive_string *, 172 const void *, size_t, struct archive_string_conv *); 173 static int best_effort_strncat_to_utf16le(struct archive_string *, 174 const void *, size_t, struct archive_string_conv *); 175 #if defined(HAVE_ICONV) 176 static int iconv_strncat_in_locale(struct archive_string *, const void *, 177 size_t, struct archive_string_conv *); 178 #endif 179 static int best_effort_strncat_in_locale(struct archive_string *, 180 const void *, size_t, struct archive_string_conv *); 181 static int _utf8_to_unicode(uint32_t *, const char *, size_t); 182 static int utf8_to_unicode(uint32_t *, const char *, size_t); 183 static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t); 184 static int cesu8_to_unicode(uint32_t *, const char *, size_t); 185 static size_t unicode_to_utf8(char *, size_t, uint32_t); 186 static int utf16_to_unicode(uint32_t *, const char *, size_t, int); 187 static size_t unicode_to_utf16be(char *, size_t, uint32_t); 188 static size_t unicode_to_utf16le(char *, size_t, uint32_t); 189 static int strncat_from_utf8_libarchive2(struct archive_string *, 190 const void *, size_t, struct archive_string_conv *); 191 static int strncat_from_utf8_to_utf8(struct archive_string *, const void *, 192 size_t, struct archive_string_conv *); 193 static int archive_string_normalize_C(struct archive_string *, const void *, 194 size_t, struct archive_string_conv *); 195 static int archive_string_normalize_D(struct archive_string *, const void *, 196 size_t, struct archive_string_conv *); 197 static int archive_string_append_unicode(struct archive_string *, 198 const void *, size_t, struct archive_string_conv *); 199 200 static struct archive_string * 201 archive_string_append(struct archive_string *as, const char *p, size_t s) 202 { 203 if (archive_string_ensure(as, as->length + s + 1) == NULL) 204 return (NULL); 205 memmove(as->s + as->length, p, s); 206 as->length += s; 207 as->s[as->length] = 0; 208 return (as); 209 } 210 211 static struct archive_wstring * 212 archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) 213 { 214 if (archive_wstring_ensure(as, as->length + s + 1) == NULL) 215 return (NULL); 216 wmemmove(as->s + as->length, p, s); 217 as->length += s; 218 as->s[as->length] = 0; 219 return (as); 220 } 221 222 void 223 archive_string_concat(struct archive_string *dest, struct archive_string *src) 224 { 225 if (archive_string_append(dest, src->s, src->length) == NULL) 226 __archive_errx(1, "Out of memory"); 227 } 228 229 void 230 archive_wstring_concat(struct archive_wstring *dest, 231 struct archive_wstring *src) 232 { 233 if (archive_wstring_append(dest, src->s, src->length) == NULL) 234 __archive_errx(1, "Out of memory"); 235 } 236 237 void 238 archive_string_free(struct archive_string *as) 239 { 240 as->length = 0; 241 as->buffer_length = 0; 242 free(as->s); 243 as->s = NULL; 244 } 245 246 void 247 archive_wstring_free(struct archive_wstring *as) 248 { 249 as->length = 0; 250 as->buffer_length = 0; 251 free(as->s); 252 as->s = NULL; 253 } 254 255 struct archive_wstring * 256 archive_wstring_ensure(struct archive_wstring *as, size_t s) 257 { 258 return (struct archive_wstring *) 259 archive_string_ensure((struct archive_string *)as, 260 s * sizeof(wchar_t)); 261 } 262 263 /* Returns NULL on any allocation failure. */ 264 struct archive_string * 265 archive_string_ensure(struct archive_string *as, size_t s) 266 { 267 char *p; 268 size_t new_length; 269 270 /* If buffer is already big enough, don't reallocate. */ 271 if (as->s && (s <= as->buffer_length)) 272 return (as); 273 274 /* 275 * Growing the buffer at least exponentially ensures that 276 * append operations are always linear in the number of 277 * characters appended. Using a smaller growth rate for 278 * larger buffers reduces memory waste somewhat at the cost of 279 * a larger constant factor. 280 */ 281 if (as->buffer_length < 32) 282 /* Start with a minimum 32-character buffer. */ 283 new_length = 32; 284 else if (as->buffer_length < 8192) 285 /* Buffers under 8k are doubled for speed. */ 286 new_length = as->buffer_length + as->buffer_length; 287 else { 288 /* Buffers 8k and over grow by at least 25% each time. */ 289 new_length = as->buffer_length + as->buffer_length / 4; 290 /* Be safe: If size wraps, fail. */ 291 if (new_length < as->buffer_length) { 292 /* On failure, wipe the string and return NULL. */ 293 archive_string_free(as); 294 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 295 return (NULL); 296 } 297 } 298 /* 299 * The computation above is a lower limit to how much we'll 300 * grow the buffer. In any case, we have to grow it enough to 301 * hold the request. 302 */ 303 if (new_length < s) 304 new_length = s; 305 /* Now we can reallocate the buffer. */ 306 p = (char *)realloc(as->s, new_length); 307 if (p == NULL) { 308 /* On failure, wipe the string and return NULL. */ 309 archive_string_free(as); 310 errno = ENOMEM;/* Make sure errno has ENOMEM. */ 311 return (NULL); 312 } 313 314 as->s = p; 315 as->buffer_length = new_length; 316 return (as); 317 } 318 319 /* 320 * TODO: See if there's a way to avoid scanning 321 * the source string twice. Then test to see 322 * if it actually helps (remember that we're almost 323 * always called with pretty short arguments, so 324 * such an optimization might not help). 325 */ 326 struct archive_string * 327 archive_strncat(struct archive_string *as, const void *_p, size_t n) 328 { 329 size_t s; 330 const char *p, *pp; 331 332 p = (const char *)_p; 333 334 /* Like strlen(p), except won't examine positions beyond p[n]. */ 335 s = 0; 336 pp = p; 337 while (s < n && *pp) { 338 pp++; 339 s++; 340 } 341 if ((as = archive_string_append(as, p, s)) == NULL) 342 __archive_errx(1, "Out of memory"); 343 return (as); 344 } 345 346 struct archive_wstring * 347 archive_wstrncat(struct archive_wstring *as, const wchar_t *p, size_t n) 348 { 349 size_t s; 350 const wchar_t *pp; 351 352 /* Like strlen(p), except won't examine positions beyond p[n]. */ 353 s = 0; 354 pp = p; 355 while (s < n && *pp) { 356 pp++; 357 s++; 358 } 359 if ((as = archive_wstring_append(as, p, s)) == NULL) 360 __archive_errx(1, "Out of memory"); 361 return (as); 362 } 363 364 struct archive_string * 365 archive_strcat(struct archive_string *as, const void *p) 366 { 367 /* strcat is just strncat without an effective limit. 368 * Assert that we'll never get called with a source 369 * string over 16MB. 370 * TODO: Review all uses of strcat in the source 371 * and try to replace them with strncat(). 372 */ 373 return archive_strncat(as, p, 0x1000000); 374 } 375 376 struct archive_wstring * 377 archive_wstrcat(struct archive_wstring *as, const wchar_t *p) 378 { 379 /* Ditto. */ 380 return archive_wstrncat(as, p, 0x1000000); 381 } 382 383 struct archive_string * 384 archive_strappend_char(struct archive_string *as, char c) 385 { 386 if ((as = archive_string_append(as, &c, 1)) == NULL) 387 __archive_errx(1, "Out of memory"); 388 return (as); 389 } 390 391 struct archive_wstring * 392 archive_wstrappend_wchar(struct archive_wstring *as, wchar_t c) 393 { 394 if ((as = archive_wstring_append(as, &c, 1)) == NULL) 395 __archive_errx(1, "Out of memory"); 396 return (as); 397 } 398 399 /* 400 * Get the "current character set" name to use with iconv. 401 * On FreeBSD, the empty character set name "" chooses 402 * the correct character encoding for the current locale, 403 * so this isn't necessary. 404 * But iconv on Mac OS 10.6 doesn't seem to handle this correctly; 405 * on that system, we have to explicitly call nl_langinfo() 406 * to get the right name. Not sure about other platforms. 407 * 408 * NOTE: GNU libiconv does not recognize the character-set name 409 * which some platform nl_langinfo(CODESET) returns, so we should 410 * use locale_charset() instead of nl_langinfo(CODESET) for GNU libiconv. 411 */ 412 static const char * 413 default_iconv_charset(const char *charset) { 414 if (charset != NULL && charset[0] != '\0') 415 return charset; 416 #if HAVE_LOCALE_CHARSET && !defined(__APPLE__) 417 /* locale_charset() is broken on Mac OS */ 418 return locale_charset(); 419 #elif HAVE_NL_LANGINFO 420 return nl_langinfo(CODESET); 421 #else 422 return ""; 423 #endif 424 } 425 426 #if defined(_WIN32) && !defined(__CYGWIN__) 427 428 /* 429 * Convert MBS to WCS. 430 * Note: returns -1 if conversion fails. 431 */ 432 int 433 archive_wstring_append_from_mbs(struct archive_wstring *dest, 434 const char *p, size_t len) 435 { 436 return archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL); 437 } 438 439 static int 440 archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest, 441 const char *s, size_t length, struct archive_string_conv *sc) 442 { 443 int count, ret = 0; 444 UINT from_cp; 445 446 if (sc != NULL) 447 from_cp = sc->from_cp; 448 else 449 from_cp = get_current_codepage(); 450 451 if (from_cp == CP_C_LOCALE) { 452 /* 453 * "C" locale special process. 454 */ 455 wchar_t *ws; 456 const unsigned char *mp; 457 458 if (NULL == archive_wstring_ensure(dest, 459 dest->length + length + 1)) 460 return (-1); 461 462 ws = dest->s + dest->length; 463 mp = (const unsigned char *)s; 464 count = 0; 465 while (count < (int)length && *mp) { 466 *ws++ = (wchar_t)*mp++; 467 count++; 468 } 469 } else if (sc != NULL && 470 (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) { 471 /* 472 * Normalize UTF-8 and UTF-16BE and convert it directly 473 * to UTF-16 as wchar_t. 474 */ 475 struct archive_string u16; 476 int saved_flag = sc->flag;/* save current flag. */ 477 478 if (is_big_endian()) 479 sc->flag |= SCONV_TO_UTF16BE; 480 else 481 sc->flag |= SCONV_TO_UTF16LE; 482 483 if (sc->flag & SCONV_FROM_UTF16) { 484 /* 485 * UTF-16BE/LE NFD ===> UTF-16 NFC 486 * UTF-16BE/LE NFC ===> UTF-16 NFD 487 */ 488 count = (int)utf16nbytes(s, length); 489 } else { 490 /* 491 * UTF-8 NFD ===> UTF-16 NFC 492 * UTF-8 NFC ===> UTF-16 NFD 493 */ 494 count = (int)mbsnbytes(s, length); 495 } 496 u16.s = (char *)dest->s; 497 u16.length = dest->length << 1;; 498 u16.buffer_length = dest->buffer_length; 499 if (sc->flag & SCONV_NORMALIZATION_C) 500 ret = archive_string_normalize_C(&u16, s, count, sc); 501 else 502 ret = archive_string_normalize_D(&u16, s, count, sc); 503 dest->s = (wchar_t *)u16.s; 504 dest->length = u16.length >> 1; 505 dest->buffer_length = u16.buffer_length; 506 sc->flag = saved_flag;/* restore the saved flag. */ 507 return (ret); 508 } else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) { 509 count = (int)utf16nbytes(s, length); 510 count >>= 1; /* to be WCS length */ 511 /* Allocate memory for WCS. */ 512 if (NULL == archive_wstring_ensure(dest, 513 dest->length + count + 1)) 514 return (-1); 515 wmemcpy(dest->s + dest->length, (const wchar_t *)s, count); 516 if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) { 517 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 518 int b; 519 for (b = 0; b < count; b++) { 520 uint16_t val = archive_le16dec(u16+b); 521 archive_be16enc(u16+b, val); 522 } 523 } else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) { 524 uint16_t *u16 = (uint16_t *)(dest->s + dest->length); 525 int b; 526 for (b = 0; b < count; b++) { 527 uint16_t val = archive_be16dec(u16+b); 528 archive_le16enc(u16+b, val); 529 } 530 } 531 } else { 532 DWORD mbflag; 533 size_t buffsize; 534 535 if (sc == NULL) 536 mbflag = 0; 537 else if (sc->flag & SCONV_FROM_CHARSET) { 538 /* Do not trust the length which comes from 539 * an archive file. */ 540 length = mbsnbytes(s, length); 541 mbflag = 0; 542 } else 543 mbflag = MB_PRECOMPOSED; 544 545 buffsize = dest->length + length + 1; 546 do { 547 /* Allocate memory for WCS. */ 548 if (NULL == archive_wstring_ensure(dest, buffsize)) 549 return (-1); 550 /* Convert MBS to WCS. */ 551 count = MultiByteToWideChar(from_cp, 552 mbflag, s, (int)length, dest->s + dest->length, 553 (int)(dest->buffer_length >> 1) -1); 554 if (count == 0 && 555 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 556 /* Expand the WCS buffer. */ 557 buffsize = dest->buffer_length << 1; 558 continue; 559 } 560 if (count == 0 && length != 0) 561 ret = -1; 562 } while (0); 563 } 564 dest->length += count; 565 dest->s[dest->length] = L'\0'; 566 return (ret); 567 } 568 569 #else 570 571 /* 572 * Convert MBS to WCS. 573 * Note: returns -1 if conversion fails. 574 */ 575 int 576 archive_wstring_append_from_mbs(struct archive_wstring *dest, 577 const char *p, size_t len) 578 { 579 size_t r; 580 int ret_val = 0; 581 /* 582 * No single byte will be more than one wide character, 583 * so this length estimate will always be big enough. 584 */ 585 size_t wcs_length = len; 586 size_t mbs_length = len; 587 const char *mbs = p; 588 wchar_t *wcs; 589 #if HAVE_MBRTOWC 590 mbstate_t shift_state; 591 592 memset(&shift_state, 0, sizeof(shift_state)); 593 #endif 594 if (NULL == archive_wstring_ensure(dest, dest->length + wcs_length + 1)) 595 return (-1); 596 wcs = dest->s + dest->length; 597 /* 598 * We cannot use mbsrtowcs/mbstowcs here because those may convert 599 * extra MBS when strlen(p) > len and one wide character consis of 600 * multi bytes. 601 */ 602 while (*mbs && mbs_length > 0) { 603 if (wcs_length == 0) { 604 dest->length = wcs - dest->s; 605 dest->s[dest->length] = L'\0'; 606 wcs_length = mbs_length; 607 if (NULL == archive_wstring_ensure(dest, 608 dest->length + wcs_length + 1)) 609 return (-1); 610 wcs = dest->s + dest->length; 611 } 612 #if HAVE_MBRTOWC 613 r = mbrtowc(wcs, mbs, wcs_length, &shift_state); 614 #else 615 r = mbtowc(wcs, mbs, wcs_length); 616 #endif 617 if (r == (size_t)-1 || r == (size_t)-2) { 618 ret_val = -1; 619 if (errno == EILSEQ) { 620 ++mbs; 621 --mbs_length; 622 continue; 623 } else 624 break; 625 } 626 if (r == 0 || r > mbs_length) 627 break; 628 wcs++; 629 wcs_length--; 630 mbs += r; 631 mbs_length -= r; 632 } 633 dest->length = wcs - dest->s; 634 dest->s[dest->length] = L'\0'; 635 return (ret_val); 636 } 637 638 #endif 639 640 #if defined(_WIN32) && !defined(__CYGWIN__) 641 642 /* 643 * WCS ==> MBS. 644 * Note: returns -1 if conversion fails. 645 * 646 * Win32 builds use WideCharToMultiByte from the Windows API. 647 * (Maybe Cygwin should too? WideCharToMultiByte will know a 648 * lot more about local character encodings than the wcrtomb() 649 * wrapper is going to know.) 650 */ 651 int 652 archive_string_append_from_wcs(struct archive_string *as, 653 const wchar_t *w, size_t len) 654 { 655 return archive_string_append_from_wcs_in_codepage(as, w, len, NULL); 656 } 657 658 static int 659 archive_string_append_from_wcs_in_codepage(struct archive_string *as, 660 const wchar_t *ws, size_t len, struct archive_string_conv *sc) 661 { 662 BOOL defchar_used, *dp; 663 int count, ret = 0; 664 UINT to_cp; 665 int wslen = (int)len; 666 667 if (sc != NULL) 668 to_cp = sc->to_cp; 669 else 670 to_cp = get_current_codepage(); 671 672 if (to_cp == CP_C_LOCALE) { 673 /* 674 * "C" locale special process. 675 */ 676 const wchar_t *wp = ws; 677 char *p; 678 679 if (NULL == archive_string_ensure(as, 680 as->length + wslen +1)) 681 return (-1); 682 p = as->s + as->length; 683 count = 0; 684 defchar_used = 0; 685 while (count < wslen && *wp) { 686 if (*wp > 255) { 687 *p++ = '?'; 688 wp++; 689 defchar_used = 1; 690 } else 691 *p++ = (char)*wp++; 692 count++; 693 } 694 } else if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) { 695 uint16_t *u16; 696 697 if (NULL == 698 archive_string_ensure(as, as->length + len * 2 + 2)) 699 return (-1); 700 u16 = (uint16_t *)(as->s + as->length); 701 count = 0; 702 defchar_used = 0; 703 if (sc->flag & SCONV_TO_UTF16BE) { 704 while (count < (int)len && *ws) { 705 archive_be16enc(u16+count, *ws); 706 ws++; 707 count++; 708 } 709 } else { 710 while (count < (int)len && *ws) { 711 archive_le16enc(u16+count, *ws); 712 ws++; 713 count++; 714 } 715 } 716 count <<= 1; /* to be byte size */ 717 } else { 718 /* Make sure the MBS buffer has plenty to set. */ 719 if (NULL == 720 archive_string_ensure(as, as->length + len * 2 + 1)) 721 return (-1); 722 do { 723 defchar_used = 0; 724 if (to_cp == CP_UTF8 || sc == NULL) 725 dp = NULL; 726 else 727 dp = &defchar_used; 728 count = WideCharToMultiByte(to_cp, 0, ws, wslen, 729 as->s + as->length, (int)as->buffer_length-1, NULL, dp); 730 if (count == 0 && 731 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 732 /* Expand the MBS buffer and retry. */ 733 if (NULL == archive_string_ensure(as, 734 as->buffer_length + len)) 735 return (-1); 736 continue; 737 } 738 if (count == 0) 739 ret = -1; 740 break; 741 } while (1); 742 } 743 as->length += count; 744 as->s[as->length] = '\0'; 745 return (defchar_used?-1:ret); 746 } 747 748 #elif defined(HAVE_WCTOMB) || defined(HAVE_WCRTOMB) 749 750 /* 751 * Translates a wide character string into current locale character set 752 * and appends to the archive_string. Note: returns -1 if conversion 753 * fails. 754 */ 755 int 756 archive_string_append_from_wcs(struct archive_string *as, 757 const wchar_t *w, size_t len) 758 { 759 /* We cannot use the standard wcstombs() here because it 760 * cannot tell us how big the output buffer should be. So 761 * I've built a loop around wcrtomb() or wctomb() that 762 * converts a character at a time and resizes the string as 763 * needed. We prefer wcrtomb() when it's available because 764 * it's thread-safe. */ 765 int n, ret_val = 0; 766 char *p; 767 char *end; 768 #if HAVE_WCRTOMB 769 mbstate_t shift_state; 770 771 memset(&shift_state, 0, sizeof(shift_state)); 772 #else 773 /* Clear the shift state before starting. */ 774 wctomb(NULL, L'\0'); 775 #endif 776 /* 777 * Allocate buffer for MBS. 778 * We need this allocation here since it is possible that 779 * as->s is still NULL. 780 */ 781 if (archive_string_ensure(as, as->length + len + 1) == NULL) 782 return (-1); 783 784 p = as->s + as->length; 785 end = as->s + as->buffer_length - MB_CUR_MAX -1; 786 while (*w != L'\0' && len > 0) { 787 if (p >= end) { 788 as->length = p - as->s; 789 as->s[as->length] = '\0'; 790 /* Re-allocate buffer for MBS. */ 791 if (archive_string_ensure(as, 792 as->length + len * 2 + 1) == NULL) 793 return (-1); 794 p = as->s + as->length; 795 end = as->s + as->buffer_length - MB_CUR_MAX -1; 796 } 797 #if HAVE_WCRTOMB 798 n = wcrtomb(p, *w++, &shift_state); 799 #else 800 n = wctomb(p, *w++); 801 #endif 802 if (n == -1) { 803 if (errno == EILSEQ) { 804 /* Skip an illegal wide char. */ 805 *p++ = '?'; 806 ret_val = -1; 807 } else { 808 ret_val = -1; 809 break; 810 } 811 } else 812 p += n; 813 len--; 814 } 815 as->length = p - as->s; 816 as->s[as->length] = '\0'; 817 return (ret_val); 818 } 819 820 #else /* HAVE_WCTOMB || HAVE_WCRTOMB */ 821 822 /* 823 * TODO: Test if __STDC_ISO_10646__ is defined. 824 * Non-Windows uses ISO C wcrtomb() or wctomb() to perform the conversion 825 * one character at a time. If a non-Windows platform doesn't have 826 * either of these, fall back to the built-in UTF8 conversion. 827 */ 828 int 829 archive_string_append_from_wcs(struct archive_string *as, 830 const wchar_t *w, size_t len) 831 { 832 (void)as;/* UNUSED */ 833 (void)w;/* UNUSED */ 834 (void)len;/* UNUSED */ 835 errno = ENOSYS; 836 return (-1); 837 } 838 839 #endif /* HAVE_WCTOMB || HAVE_WCRTOMB */ 840 841 /* 842 * Find a string conversion object by a pair of 'from' charset name 843 * and 'to' charset name from an archive object. 844 * Return NULL if not found. 845 */ 846 static struct archive_string_conv * 847 find_sconv_object(struct archive *a, const char *fc, const char *tc) 848 { 849 struct archive_string_conv *sc; 850 851 if (a == NULL) 852 return (NULL); 853 854 for (sc = a->sconv; sc != NULL; sc = sc->next) { 855 if (strcmp(sc->from_charset, fc) == 0 && 856 strcmp(sc->to_charset, tc) == 0) 857 break; 858 } 859 return (sc); 860 } 861 862 /* 863 * Register a string object to an archive object. 864 */ 865 static void 866 add_sconv_object(struct archive *a, struct archive_string_conv *sc) 867 { 868 struct archive_string_conv **psc; 869 870 /* Add a new sconv to sconv list. */ 871 psc = &(a->sconv); 872 while (*psc != NULL) 873 psc = &((*psc)->next); 874 *psc = sc; 875 } 876 877 static void 878 add_converter(struct archive_string_conv *sc, int (*converter) 879 (struct archive_string *, const void *, size_t, 880 struct archive_string_conv *)) 881 { 882 if (sc == NULL || sc->nconverter >= 2) 883 __archive_errx(1, "Programing error"); 884 sc->converter[sc->nconverter++] = converter; 885 } 886 887 static void 888 setup_converter(struct archive_string_conv *sc) 889 { 890 891 /* Reset. */ 892 sc->nconverter = 0; 893 894 /* 895 * Perform special sequence for the incorrect UTF-8 filenames 896 * made by libarchive2.x. 897 */ 898 if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) { 899 add_converter(sc, strncat_from_utf8_libarchive2); 900 return; 901 } 902 903 /* 904 * Convert a string to UTF-16BE/LE. 905 */ 906 if (sc->flag & SCONV_TO_UTF16) { 907 /* 908 * If the current locale is UTF-8, we can translate 909 * a UTF-8 string into a UTF-16BE string. 910 */ 911 if (sc->flag & SCONV_FROM_UTF8) { 912 add_converter(sc, archive_string_append_unicode); 913 return; 914 } 915 916 #if defined(_WIN32) && !defined(__CYGWIN__) 917 if (sc->flag & SCONV_WIN_CP) { 918 if (sc->flag & SCONV_TO_UTF16BE) 919 add_converter(sc, win_strncat_to_utf16be); 920 else 921 add_converter(sc, win_strncat_to_utf16le); 922 return; 923 } 924 #endif 925 926 #if defined(HAVE_ICONV) 927 if (sc->cd != (iconv_t)-1) { 928 add_converter(sc, iconv_strncat_in_locale); 929 return; 930 } 931 #endif 932 933 if (sc->flag & SCONV_BEST_EFFORT) { 934 if (sc->flag & SCONV_TO_UTF16BE) 935 add_converter(sc, 936 best_effort_strncat_to_utf16be); 937 else 938 add_converter(sc, 939 best_effort_strncat_to_utf16le); 940 } else 941 /* Make sure we have no converter. */ 942 sc->nconverter = 0; 943 return; 944 } 945 946 /* 947 * Convert a string from UTF-16BE/LE. 948 */ 949 if (sc->flag & SCONV_FROM_UTF16) { 950 /* 951 * At least we should normalize a UTF-16BE string. 952 */ 953 if (sc->flag & SCONV_NORMALIZATION_D) 954 add_converter(sc,archive_string_normalize_D); 955 else if (sc->flag & SCONV_NORMALIZATION_C) 956 add_converter(sc, archive_string_normalize_C); 957 958 if (sc->flag & SCONV_TO_UTF8) { 959 /* 960 * If the current locale is UTF-8, we can translate 961 * a UTF-16BE/LE string into a UTF-8 string directly. 962 */ 963 if (!(sc->flag & 964 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 965 add_converter(sc, 966 archive_string_append_unicode); 967 return; 968 } 969 970 #if defined(_WIN32) && !defined(__CYGWIN__) 971 if (sc->flag & SCONV_WIN_CP) { 972 if (sc->flag & SCONV_FROM_UTF16BE) 973 add_converter(sc, win_strncat_from_utf16be); 974 else 975 add_converter(sc, win_strncat_from_utf16le); 976 return; 977 } 978 #endif 979 980 #if defined(HAVE_ICONV) 981 if (sc->cd != (iconv_t)-1) { 982 add_converter(sc, iconv_strncat_in_locale); 983 return; 984 } 985 #endif 986 987 if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 988 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16BE)) 989 add_converter(sc, best_effort_strncat_from_utf16be); 990 else if ((sc->flag & (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 991 == (SCONV_BEST_EFFORT | SCONV_FROM_UTF16LE)) 992 add_converter(sc, best_effort_strncat_from_utf16le); 993 else 994 /* Make sure we have no converter. */ 995 sc->nconverter = 0; 996 return; 997 } 998 999 if (sc->flag & SCONV_FROM_UTF8) { 1000 /* 1001 * At least we should normalize a UTF-8 string. 1002 */ 1003 if (sc->flag & SCONV_NORMALIZATION_D) 1004 add_converter(sc,archive_string_normalize_D); 1005 else if (sc->flag & SCONV_NORMALIZATION_C) 1006 add_converter(sc, archive_string_normalize_C); 1007 1008 /* 1009 * Copy UTF-8 string with a check of CESU-8. 1010 * Apparently, iconv does not check surrogate pairs in UTF-8 1011 * when both from-charset and to-charset are UTF-8, and then 1012 * we use our UTF-8 copy code. 1013 */ 1014 if (sc->flag & SCONV_TO_UTF8) { 1015 /* 1016 * If the current locale is UTF-8, we can translate 1017 * a UTF-16BE string into a UTF-8 string directly. 1018 */ 1019 if (!(sc->flag & 1020 (SCONV_NORMALIZATION_D |SCONV_NORMALIZATION_C))) 1021 add_converter(sc, strncat_from_utf8_to_utf8); 1022 return; 1023 } 1024 } 1025 1026 #if defined(_WIN32) && !defined(__CYGWIN__) 1027 /* 1028 * On Windows we can use Windows API for a string conversion. 1029 */ 1030 if (sc->flag & SCONV_WIN_CP) { 1031 add_converter(sc, strncat_in_codepage); 1032 return; 1033 } 1034 #endif 1035 1036 #if HAVE_ICONV 1037 if (sc->cd != (iconv_t)-1) { 1038 add_converter(sc, iconv_strncat_in_locale); 1039 /* 1040 * iconv generally does not support UTF-8-MAC and so 1041 * we have to the output of iconv from NFC to NFD if 1042 * need. 1043 */ 1044 if ((sc->flag & SCONV_FROM_CHARSET) && 1045 (sc->flag & SCONV_TO_UTF8)) { 1046 if (sc->flag & SCONV_NORMALIZATION_D) 1047 add_converter(sc, archive_string_normalize_D); 1048 } 1049 return; 1050 } 1051 #endif 1052 1053 /* 1054 * Try conversion in the best effort or no conversion. 1055 */ 1056 if ((sc->flag & SCONV_BEST_EFFORT) || sc->same) 1057 add_converter(sc, best_effort_strncat_in_locale); 1058 else 1059 /* Make sure we have no converter. */ 1060 sc->nconverter = 0; 1061 } 1062 1063 /* 1064 * Return canonicalized charset-name but this supports just UTF-8, UTF-16BE 1065 * and CP932 which are referenced in create_sconv_object(). 1066 */ 1067 static const char * 1068 canonical_charset_name(const char *charset) 1069 { 1070 char cs[16]; 1071 char *p; 1072 const char *s; 1073 1074 if (charset == NULL || charset[0] == '\0' 1075 || strlen(charset) > 15) 1076 return (charset); 1077 1078 /* Copy name to uppercase. */ 1079 p = cs; 1080 s = charset; 1081 while (*s) { 1082 char c = *s++; 1083 if (c >= 'a' && c <= 'z') 1084 c -= 'a' - 'A'; 1085 *p++ = c; 1086 } 1087 *p++ = '\0'; 1088 1089 if (strcmp(cs, "UTF-8") == 0 || 1090 strcmp(cs, "UTF8") == 0) 1091 return ("UTF-8"); 1092 if (strcmp(cs, "UTF-16BE") == 0 || 1093 strcmp(cs, "UTF16BE") == 0) 1094 return ("UTF-16BE"); 1095 if (strcmp(cs, "UTF-16LE") == 0 || 1096 strcmp(cs, "UTF16LE") == 0) 1097 return ("UTF-16LE"); 1098 if (strcmp(cs, "CP932") == 0) 1099 return ("CP932"); 1100 return (charset); 1101 } 1102 1103 /* 1104 * Create a string conversion object. 1105 */ 1106 static struct archive_string_conv * 1107 create_sconv_object(const char *fc, const char *tc, 1108 unsigned current_codepage, int flag) 1109 { 1110 struct archive_string_conv *sc; 1111 1112 sc = calloc(1, sizeof(*sc)); 1113 if (sc == NULL) 1114 return (NULL); 1115 sc->next = NULL; 1116 sc->from_charset = strdup(fc); 1117 if (sc->from_charset == NULL) { 1118 free(sc); 1119 return (NULL); 1120 } 1121 sc->to_charset = strdup(tc); 1122 if (sc->to_charset == NULL) { 1123 free(sc->from_charset); 1124 free(sc); 1125 return (NULL); 1126 } 1127 archive_string_init(&sc->utftmp); 1128 1129 if (flag & SCONV_TO_CHARSET) { 1130 /* 1131 * Convert characters from the current locale charset to 1132 * a specified charset. 1133 */ 1134 sc->from_cp = current_codepage; 1135 sc->to_cp = make_codepage_from_charset(tc); 1136 #if defined(_WIN32) && !defined(__CYGWIN__) 1137 if (IsValidCodePage(sc->to_cp)) 1138 flag |= SCONV_WIN_CP; 1139 #endif 1140 } else if (flag & SCONV_FROM_CHARSET) { 1141 /* 1142 * Convert characters from a specified charset to 1143 * the current locale charset. 1144 */ 1145 sc->to_cp = current_codepage; 1146 sc->from_cp = make_codepage_from_charset(fc); 1147 #if defined(_WIN32) && !defined(__CYGWIN__) 1148 if (IsValidCodePage(sc->from_cp)) 1149 flag |= SCONV_WIN_CP; 1150 #endif 1151 } 1152 1153 /* 1154 * Check if "from charset" and "to charset" are the same. 1155 */ 1156 if (strcmp(fc, tc) == 0 || 1157 (sc->from_cp != (unsigned)-1 && sc->from_cp == sc->to_cp)) 1158 sc->same = 1; 1159 else 1160 sc->same = 0; 1161 1162 /* 1163 * Mark if "from charset" or "to charset" are UTF-8 or UTF-16BE/LE. 1164 */ 1165 if (strcmp(tc, "UTF-8") == 0) 1166 flag |= SCONV_TO_UTF8; 1167 else if (strcmp(tc, "UTF-16BE") == 0) 1168 flag |= SCONV_TO_UTF16BE; 1169 else if (strcmp(tc, "UTF-16LE") == 0) 1170 flag |= SCONV_TO_UTF16LE; 1171 if (strcmp(fc, "UTF-8") == 0) 1172 flag |= SCONV_FROM_UTF8; 1173 else if (strcmp(fc, "UTF-16BE") == 0) 1174 flag |= SCONV_FROM_UTF16BE; 1175 else if (strcmp(fc, "UTF-16LE") == 0) 1176 flag |= SCONV_FROM_UTF16LE; 1177 #if defined(_WIN32) && !defined(__CYGWIN__) 1178 if (sc->to_cp == CP_UTF8) 1179 flag |= SCONV_TO_UTF8; 1180 else if (sc->to_cp == CP_UTF16BE) 1181 flag |= SCONV_TO_UTF16BE | SCONV_WIN_CP; 1182 else if (sc->to_cp == CP_UTF16LE) 1183 flag |= SCONV_TO_UTF16LE | SCONV_WIN_CP; 1184 if (sc->from_cp == CP_UTF8) 1185 flag |= SCONV_FROM_UTF8; 1186 else if (sc->from_cp == CP_UTF16BE) 1187 flag |= SCONV_FROM_UTF16BE | SCONV_WIN_CP; 1188 else if (sc->from_cp == CP_UTF16LE) 1189 flag |= SCONV_FROM_UTF16LE | SCONV_WIN_CP; 1190 #endif 1191 1192 /* 1193 * Set a flag for Unicode NFD. Usually iconv cannot correctly 1194 * handle it. So we have to translate NFD characters to NFC ones 1195 * ourselves before iconv handles. Another reason is to prevent 1196 * that the same sight of two filenames, one is NFC and other 1197 * is NFD, would be in its directory. 1198 * On Mac OS X, although its filesystem layer automatically 1199 * convert filenames to NFD, it would be useful for filename 1200 * comparing to find out the same filenames that we normalize 1201 * that to be NFD ourselves. 1202 */ 1203 if ((flag & SCONV_FROM_CHARSET) && 1204 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8))) { 1205 #if defined(__APPLE__) 1206 if (flag & SCONV_TO_UTF8) 1207 flag |= SCONV_NORMALIZATION_D; 1208 else 1209 #endif 1210 flag |= SCONV_NORMALIZATION_C; 1211 } 1212 #if defined(__APPLE__) 1213 /* 1214 * In case writing an archive file, make sure that a filename 1215 * going to be passed to iconv is a Unicode NFC string since 1216 * a filename in HFS Plus filesystem is a Unicode NFD one and 1217 * iconv cannot handle it with "UTF-8" charset. It is simpler 1218 * than a use of "UTF-8-MAC" charset. 1219 */ 1220 if ((flag & SCONV_TO_CHARSET) && 1221 (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1222 !(flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1223 flag |= SCONV_NORMALIZATION_C; 1224 /* 1225 * In case reading an archive file. make sure that a filename 1226 * will be passed to users is a Unicode NFD string in order to 1227 * correctly compare the filename with other one which comes 1228 * from HFS Plus filesystem. 1229 */ 1230 if ((flag & SCONV_FROM_CHARSET) && 1231 !(flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1232 (flag & SCONV_TO_UTF8)) 1233 flag |= SCONV_NORMALIZATION_D; 1234 #endif 1235 1236 #if defined(HAVE_ICONV) 1237 sc->cd_w = (iconv_t)-1; 1238 /* 1239 * Create an iconv object. 1240 */ 1241 if (((flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) && 1242 (flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16))) || 1243 (flag & SCONV_WIN_CP)) { 1244 /* This case we won't use iconv. */ 1245 sc->cd = (iconv_t)-1; 1246 } else { 1247 sc->cd = iconv_open(tc, fc); 1248 if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT)) { 1249 /* 1250 * Unfortunaly, all of iconv implements do support 1251 * "CP932" character-set, so we should use "SJIS" 1252 * instead if iconv_open failed. 1253 */ 1254 if (strcmp(tc, "CP932") == 0) 1255 sc->cd = iconv_open("SJIS", fc); 1256 else if (strcmp(fc, "CP932") == 0) 1257 sc->cd = iconv_open(tc, "SJIS"); 1258 } 1259 #if defined(_WIN32) && !defined(__CYGWIN__) 1260 /* 1261 * archive_mstring on Windows directly convert multi-bytes 1262 * into archive_wstring in order not to depend on locale 1263 * so that you can do a I18N programing. This will be 1264 * used only in archive_mstring_copy_mbs_len_l so far. 1265 */ 1266 if (flag & SCONV_FROM_CHARSET) { 1267 sc->cd_w = iconv_open("UTF-8", fc); 1268 if (sc->cd_w == (iconv_t)-1 && 1269 (sc->flag & SCONV_BEST_EFFORT)) { 1270 if (strcmp(fc, "CP932") == 0) 1271 sc->cd_w = iconv_open("UTF-8", "SJIS"); 1272 } 1273 } 1274 #endif /* _WIN32 && !__CYGWIN__ */ 1275 } 1276 #endif /* HAVE_ICONV */ 1277 1278 sc->flag = flag; 1279 1280 /* 1281 * Set up converters. 1282 */ 1283 setup_converter(sc); 1284 1285 return (sc); 1286 } 1287 1288 /* 1289 * Free a string conversion object. 1290 */ 1291 static void 1292 free_sconv_object(struct archive_string_conv *sc) 1293 { 1294 free(sc->from_charset); 1295 free(sc->to_charset); 1296 archive_string_free(&sc->utftmp); 1297 #if HAVE_ICONV 1298 if (sc->cd != (iconv_t)-1) 1299 iconv_close(sc->cd); 1300 if (sc->cd_w != (iconv_t)-1) 1301 iconv_close(sc->cd_w); 1302 #endif 1303 free(sc); 1304 } 1305 1306 #if defined(_WIN32) && !defined(__CYGWIN__) 1307 static unsigned 1308 my_atoi(const char *p) 1309 { 1310 unsigned cp; 1311 1312 cp = 0; 1313 while (*p) { 1314 if (*p >= '0' && *p <= '9') 1315 cp = cp * 10 + (*p - '0'); 1316 else 1317 return (-1); 1318 p++; 1319 } 1320 return (cp); 1321 } 1322 1323 /* 1324 * Translate Charset name (as used by iconv) into CodePage (as used by Windows) 1325 * Return -1 if failed. 1326 * 1327 * Note: This translation code may be insufficient. 1328 */ 1329 static struct charset { 1330 const char *name; 1331 unsigned cp; 1332 } charsets[] = { 1333 /* MUST BE SORTED! */ 1334 {"ASCII", 1252}, 1335 {"ASMO-708", 708}, 1336 {"BIG5", 950}, 1337 {"CHINESE", 936}, 1338 {"CP367", 1252}, 1339 {"CP819", 1252}, 1340 {"CP1025", 21025}, 1341 {"DOS-720", 720}, 1342 {"DOS-862", 862}, 1343 {"EUC-CN", 51936}, 1344 {"EUC-JP", 51932}, 1345 {"EUC-KR", 949}, 1346 {"EUCCN", 51936}, 1347 {"EUCJP", 51932}, 1348 {"EUCKR", 949}, 1349 {"GB18030", 54936}, 1350 {"GB2312", 936}, 1351 {"HEBREW", 1255}, 1352 {"HZ-GB-2312", 52936}, 1353 {"IBM273", 20273}, 1354 {"IBM277", 20277}, 1355 {"IBM278", 20278}, 1356 {"IBM280", 20280}, 1357 {"IBM284", 20284}, 1358 {"IBM285", 20285}, 1359 {"IBM290", 20290}, 1360 {"IBM297", 20297}, 1361 {"IBM367", 1252}, 1362 {"IBM420", 20420}, 1363 {"IBM423", 20423}, 1364 {"IBM424", 20424}, 1365 {"IBM819", 1252}, 1366 {"IBM871", 20871}, 1367 {"IBM880", 20880}, 1368 {"IBM905", 20905}, 1369 {"IBM924", 20924}, 1370 {"ISO-8859-1", 28591}, 1371 {"ISO-8859-13", 28603}, 1372 {"ISO-8859-15", 28605}, 1373 {"ISO-8859-2", 28592}, 1374 {"ISO-8859-3", 28593}, 1375 {"ISO-8859-4", 28594}, 1376 {"ISO-8859-5", 28595}, 1377 {"ISO-8859-6", 28596}, 1378 {"ISO-8859-7", 28597}, 1379 {"ISO-8859-8", 28598}, 1380 {"ISO-8859-9", 28599}, 1381 {"ISO8859-1", 28591}, 1382 {"ISO8859-13", 28603}, 1383 {"ISO8859-15", 28605}, 1384 {"ISO8859-2", 28592}, 1385 {"ISO8859-3", 28593}, 1386 {"ISO8859-4", 28594}, 1387 {"ISO8859-5", 28595}, 1388 {"ISO8859-6", 28596}, 1389 {"ISO8859-7", 28597}, 1390 {"ISO8859-8", 28598}, 1391 {"ISO8859-9", 28599}, 1392 {"JOHAB", 1361}, 1393 {"KOI8-R", 20866}, 1394 {"KOI8-U", 21866}, 1395 {"KS_C_5601-1987", 949}, 1396 {"LATIN1", 1252}, 1397 {"LATIN2", 28592}, 1398 {"MACINTOSH", 10000}, 1399 {"SHIFT-JIS", 932}, 1400 {"SHIFT_JIS", 932}, 1401 {"SJIS", 932}, 1402 {"US", 1252}, 1403 {"US-ASCII", 1252}, 1404 {"UTF-16", 1200}, 1405 {"UTF-16BE", 1201}, 1406 {"UTF-16LE", 1200}, 1407 {"UTF-8", CP_UTF8}, 1408 {"X-EUROPA", 29001}, 1409 {"X-MAC-ARABIC", 10004}, 1410 {"X-MAC-CE", 10029}, 1411 {"X-MAC-CHINESEIMP", 10008}, 1412 {"X-MAC-CHINESETRAD", 10002}, 1413 {"X-MAC-CROATIAN", 10082}, 1414 {"X-MAC-CYRILLIC", 10007}, 1415 {"X-MAC-GREEK", 10006}, 1416 {"X-MAC-HEBREW", 10005}, 1417 {"X-MAC-ICELANDIC", 10079}, 1418 {"X-MAC-JAPANESE", 10001}, 1419 {"X-MAC-KOREAN", 10003}, 1420 {"X-MAC-ROMANIAN", 10010}, 1421 {"X-MAC-THAI", 10021}, 1422 {"X-MAC-TURKISH", 10081}, 1423 {"X-MAC-UKRAINIAN", 10017}, 1424 }; 1425 static unsigned 1426 make_codepage_from_charset(const char *charset) 1427 { 1428 char cs[16]; 1429 char *p; 1430 unsigned cp; 1431 int a, b; 1432 1433 if (charset == NULL || strlen(charset) > 15) 1434 return -1; 1435 1436 /* Copy name to uppercase. */ 1437 p = cs; 1438 while (*charset) { 1439 char c = *charset++; 1440 if (c >= 'a' && c <= 'z') 1441 c -= 'a' - 'A'; 1442 *p++ = c; 1443 } 1444 *p++ = '\0'; 1445 cp = -1; 1446 1447 /* Look it up in the table first, so that we can easily 1448 * override CP367, which we map to 1252 instead of 367. */ 1449 a = 0; 1450 b = sizeof(charsets)/sizeof(charsets[0]); 1451 while (b > a) { 1452 int c = (b + a) / 2; 1453 int r = strcmp(charsets[c].name, cs); 1454 if (r < 0) 1455 a = c + 1; 1456 else if (r > 0) 1457 b = c; 1458 else 1459 return charsets[c].cp; 1460 } 1461 1462 /* If it's not in the table, try to parse it. */ 1463 switch (*cs) { 1464 case 'C': 1465 if (cs[1] == 'P' && cs[2] >= '0' && cs[2] <= '9') { 1466 cp = my_atoi(cs + 2); 1467 } else if (strcmp(cs, "CP_ACP") == 0) 1468 cp = get_current_codepage(); 1469 else if (strcmp(cs, "CP_OEMCP") == 0) 1470 cp = get_current_oemcp(); 1471 break; 1472 case 'I': 1473 if (cs[1] == 'B' && cs[2] == 'M' && 1474 cs[3] >= '0' && cs[3] <= '9') { 1475 cp = my_atoi(cs + 3); 1476 } 1477 break; 1478 case 'W': 1479 if (strncmp(cs, "WINDOWS-", 8) == 0) { 1480 cp = my_atoi(cs + 8); 1481 if (cp != 874 && (cp < 1250 || cp > 1258)) 1482 cp = -1;/* This may invalid code. */ 1483 } 1484 break; 1485 } 1486 return (cp); 1487 } 1488 1489 /* 1490 * Return ANSI Code Page of current locale set by setlocale(). 1491 */ 1492 static unsigned 1493 get_current_codepage(void) 1494 { 1495 char *locale, *p; 1496 unsigned cp; 1497 1498 locale = setlocale(LC_CTYPE, NULL); 1499 if (locale == NULL) 1500 return (GetACP()); 1501 if (locale[0] == 'C' && locale[1] == '\0') 1502 return (CP_C_LOCALE); 1503 p = strrchr(locale, '.'); 1504 if (p == NULL) 1505 return (GetACP()); 1506 cp = my_atoi(p+1); 1507 if (cp <= 0) 1508 return (GetACP()); 1509 return (cp); 1510 } 1511 1512 /* 1513 * Translation table between Locale Name and ACP/OEMCP. 1514 */ 1515 static struct { 1516 unsigned acp; 1517 unsigned ocp; 1518 const char *locale; 1519 } acp_ocp_map[] = { 1520 { 950, 950, "Chinese_Taiwan" }, 1521 { 936, 936, "Chinese_People's Republic of China" }, 1522 { 950, 950, "Chinese_Taiwan" }, 1523 { 1250, 852, "Czech_Czech Republic" }, 1524 { 1252, 850, "Danish_Denmark" }, 1525 { 1252, 850, "Dutch_Netherlands" }, 1526 { 1252, 850, "Dutch_Belgium" }, 1527 { 1252, 437, "English_United States" }, 1528 { 1252, 850, "English_Australia" }, 1529 { 1252, 850, "English_Canada" }, 1530 { 1252, 850, "English_New Zealand" }, 1531 { 1252, 850, "English_United Kingdom" }, 1532 { 1252, 437, "English_United States" }, 1533 { 1252, 850, "Finnish_Finland" }, 1534 { 1252, 850, "French_France" }, 1535 { 1252, 850, "French_Belgium" }, 1536 { 1252, 850, "French_Canada" }, 1537 { 1252, 850, "French_Switzerland" }, 1538 { 1252, 850, "German_Germany" }, 1539 { 1252, 850, "German_Austria" }, 1540 { 1252, 850, "German_Switzerland" }, 1541 { 1253, 737, "Greek_Greece" }, 1542 { 1250, 852, "Hungarian_Hungary" }, 1543 { 1252, 850, "Icelandic_Iceland" }, 1544 { 1252, 850, "Italian_Italy" }, 1545 { 1252, 850, "Italian_Switzerland" }, 1546 { 932, 932, "Japanese_Japan" }, 1547 { 949, 949, "Korean_Korea" }, 1548 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1549 { 1252, 850, "Norwegian (BokmOl)_Norway" }, 1550 { 1252, 850, "Norwegian-Nynorsk_Norway" }, 1551 { 1250, 852, "Polish_Poland" }, 1552 { 1252, 850, "Portuguese_Portugal" }, 1553 { 1252, 850, "Portuguese_Brazil" }, 1554 { 1251, 866, "Russian_Russia" }, 1555 { 1250, 852, "Slovak_Slovakia" }, 1556 { 1252, 850, "Spanish_Spain" }, 1557 { 1252, 850, "Spanish_Mexico" }, 1558 { 1252, 850, "Spanish_Spain" }, 1559 { 1252, 850, "Swedish_Sweden" }, 1560 { 1254, 857, "Turkish_Turkey" }, 1561 { 0, 0, NULL} 1562 }; 1563 1564 /* 1565 * Return OEM Code Page of current locale set by setlocale(). 1566 */ 1567 static unsigned 1568 get_current_oemcp(void) 1569 { 1570 int i; 1571 char *locale, *p; 1572 size_t len; 1573 1574 locale = setlocale(LC_CTYPE, NULL); 1575 if (locale == NULL) 1576 return (GetOEMCP()); 1577 if (locale[0] == 'C' && locale[1] == '\0') 1578 return (CP_C_LOCALE); 1579 1580 p = strrchr(locale, '.'); 1581 if (p == NULL) 1582 return (GetOEMCP()); 1583 len = p - locale; 1584 for (i = 0; acp_ocp_map[i].acp; i++) { 1585 if (strncmp(acp_ocp_map[i].locale, locale, len) == 0) 1586 return (acp_ocp_map[i].ocp); 1587 } 1588 return (GetOEMCP()); 1589 } 1590 #else 1591 1592 /* 1593 * POSIX platform does not use CodePage. 1594 */ 1595 1596 static unsigned 1597 get_current_codepage(void) 1598 { 1599 return (-1);/* Unknown */ 1600 } 1601 static unsigned 1602 make_codepage_from_charset(const char *charset) 1603 { 1604 (void)charset; /* UNUSED */ 1605 return (-1);/* Unknown */ 1606 } 1607 static unsigned 1608 get_current_oemcp(void) 1609 { 1610 return (-1);/* Unknown */ 1611 } 1612 1613 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 1614 1615 /* 1616 * Return a string conversion object. 1617 */ 1618 static struct archive_string_conv * 1619 get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag) 1620 { 1621 struct archive_string_conv *sc; 1622 unsigned current_codepage; 1623 1624 /* Check if we have made the sconv object. */ 1625 sc = find_sconv_object(a, fc, tc); 1626 if (sc != NULL) 1627 return (sc); 1628 1629 if (a == NULL) 1630 current_codepage = get_current_codepage(); 1631 else 1632 current_codepage = a->current_codepage; 1633 1634 sc = create_sconv_object(canonical_charset_name(fc), 1635 canonical_charset_name(tc), current_codepage, flag); 1636 if (sc == NULL) { 1637 if (a != NULL) 1638 archive_set_error(a, ENOMEM, 1639 "Could not allocate memory for " 1640 "a string conversion object"); 1641 return (NULL); 1642 } 1643 1644 /* 1645 * If there is no converter for current string conversion object, 1646 * we cannot handle this conversion. 1647 */ 1648 if (sc->nconverter == 0) { 1649 if (a != NULL) { 1650 #if HAVE_ICONV 1651 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1652 "iconv_open failed : Cannot handle ``%s''", 1653 (flag & SCONV_TO_CHARSET)?tc:fc); 1654 #else 1655 archive_set_error(a, ARCHIVE_ERRNO_MISC, 1656 "A character-set conversion not fully supported " 1657 "on this platform"); 1658 #endif 1659 } 1660 /* Failed; free a sconv object. */ 1661 free_sconv_object(sc); 1662 return (NULL); 1663 } 1664 1665 /* 1666 * Success! 1667 */ 1668 if (a != NULL) 1669 add_sconv_object(a, sc); 1670 return (sc); 1671 } 1672 1673 static const char * 1674 get_current_charset(struct archive *a) 1675 { 1676 const char *cur_charset; 1677 1678 if (a == NULL) 1679 cur_charset = default_iconv_charset(""); 1680 else { 1681 cur_charset = default_iconv_charset(a->current_code); 1682 if (a->current_code == NULL) { 1683 a->current_code = strdup(cur_charset); 1684 a->current_codepage = get_current_codepage(); 1685 a->current_oemcp = get_current_oemcp(); 1686 } 1687 } 1688 return (cur_charset); 1689 } 1690 1691 /* 1692 * Make and Return a string conversion object. 1693 * Return NULL if the platform does not support the specified conversion 1694 * and best_effort is 0. 1695 * If best_effort is set, A string conversion object must be returned 1696 * unless memory allocation for the object fails, but the conversion 1697 * might fail when non-ASCII code is found. 1698 */ 1699 struct archive_string_conv * 1700 archive_string_conversion_to_charset(struct archive *a, const char *charset, 1701 int best_effort) 1702 { 1703 int flag = SCONV_TO_CHARSET; 1704 1705 if (best_effort) 1706 flag |= SCONV_BEST_EFFORT; 1707 return (get_sconv_object(a, get_current_charset(a), charset, flag)); 1708 } 1709 1710 struct archive_string_conv * 1711 archive_string_conversion_from_charset(struct archive *a, const char *charset, 1712 int best_effort) 1713 { 1714 int flag = SCONV_FROM_CHARSET; 1715 1716 if (best_effort) 1717 flag |= SCONV_BEST_EFFORT; 1718 return (get_sconv_object(a, charset, get_current_charset(a), flag)); 1719 } 1720 1721 /* 1722 * archive_string_default_conversion_*_archive() are provided for Windows 1723 * platform because other archiver application use CP_OEMCP for 1724 * MultiByteToWideChar() and WideCharToMultiByte() for the filenames 1725 * in tar or zip files. But mbstowcs/wcstombs(CRT) usually use CP_ACP 1726 * unless you use setlocale(LC_ALL, ".OCP")(specify CP_OEMCP). 1727 * So we should make a string conversion between CP_ACP and CP_OEMCP 1728 * for compatibillty. 1729 */ 1730 #if defined(_WIN32) && !defined(__CYGWIN__) 1731 struct archive_string_conv * 1732 archive_string_default_conversion_for_read(struct archive *a) 1733 { 1734 const char *cur_charset = get_current_charset(a); 1735 char oemcp[16]; 1736 1737 /* NOTE: a check of cur_charset is unneeded but we need 1738 * that get_current_charset() has been surely called at 1739 * this time whatever C compiler optimized. */ 1740 if (cur_charset != NULL && 1741 (a->current_codepage == CP_C_LOCALE || 1742 a->current_codepage == a->current_oemcp)) 1743 return (NULL);/* no conversion. */ 1744 1745 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1746 /* Make sure a null termination must be set. */ 1747 oemcp[sizeof(oemcp)-1] = '\0'; 1748 return (get_sconv_object(a, oemcp, cur_charset, 1749 SCONV_FROM_CHARSET)); 1750 } 1751 1752 struct archive_string_conv * 1753 archive_string_default_conversion_for_write(struct archive *a) 1754 { 1755 const char *cur_charset = get_current_charset(a); 1756 char oemcp[16]; 1757 1758 /* NOTE: a check of cur_charset is unneeded but we need 1759 * that get_current_charset() has been surely called at 1760 * this time whatever C compiler optimized. */ 1761 if (cur_charset != NULL && 1762 (a->current_codepage == CP_C_LOCALE || 1763 a->current_codepage == a->current_oemcp)) 1764 return (NULL);/* no conversion. */ 1765 1766 _snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp); 1767 /* Make sure a null termination must be set. */ 1768 oemcp[sizeof(oemcp)-1] = '\0'; 1769 return (get_sconv_object(a, cur_charset, oemcp, 1770 SCONV_TO_CHARSET)); 1771 } 1772 #else 1773 struct archive_string_conv * 1774 archive_string_default_conversion_for_read(struct archive *a) 1775 { 1776 (void)a; /* UNUSED */ 1777 return (NULL); 1778 } 1779 1780 struct archive_string_conv * 1781 archive_string_default_conversion_for_write(struct archive *a) 1782 { 1783 (void)a; /* UNUSED */ 1784 return (NULL); 1785 } 1786 #endif 1787 1788 /* 1789 * Dispose of all character conversion objects in the archive object. 1790 */ 1791 void 1792 archive_string_conversion_free(struct archive *a) 1793 { 1794 struct archive_string_conv *sc; 1795 struct archive_string_conv *sc_next; 1796 1797 for (sc = a->sconv; sc != NULL; sc = sc_next) { 1798 sc_next = sc->next; 1799 free_sconv_object(sc); 1800 } 1801 a->sconv = NULL; 1802 free(a->current_code); 1803 a->current_code = NULL; 1804 } 1805 1806 /* 1807 * Return a conversion charset name. 1808 */ 1809 const char * 1810 archive_string_conversion_charset_name(struct archive_string_conv *sc) 1811 { 1812 if (sc->flag & SCONV_TO_CHARSET) 1813 return (sc->to_charset); 1814 else 1815 return (sc->from_charset); 1816 } 1817 1818 /* 1819 * Change the behavior of a string conversion. 1820 */ 1821 void 1822 archive_string_conversion_set_opt(struct archive_string_conv *sc, int opt) 1823 { 1824 switch (opt) { 1825 /* 1826 * A filename in UTF-8 was made with libarchive 2.x in a wrong 1827 * assumption that wchar_t was Unicode. 1828 * This option enables simulating the assumption in order to read 1829 * that filname correctly. 1830 */ 1831 case SCONV_SET_OPT_UTF8_LIBARCHIVE2X: 1832 #if (defined(_WIN32) && !defined(__CYGWIN__)) \ 1833 || defined(__STDC_ISO_10646__) || defined(__APPLE__) 1834 /* 1835 * Nothing to do for it since wchar_t on these platforms 1836 * is really Unicode. 1837 */ 1838 (void)sc; /* UNUSED */ 1839 #else 1840 if ((sc->flag & SCONV_UTF8_LIBARCHIVE_2) == 0) { 1841 sc->flag |= SCONV_UTF8_LIBARCHIVE_2; 1842 /* Set up string converters. */ 1843 setup_converter(sc); 1844 } 1845 #endif 1846 break; 1847 case SCONV_SET_OPT_NORMALIZATION_C: 1848 if ((sc->flag & SCONV_NORMALIZATION_C) == 0) { 1849 sc->flag |= SCONV_NORMALIZATION_C; 1850 sc->flag &= ~SCONV_NORMALIZATION_D; 1851 /* Set up string converters. */ 1852 setup_converter(sc); 1853 } 1854 break; 1855 case SCONV_SET_OPT_NORMALIZATION_D: 1856 #if defined(HAVE_ICONV) 1857 /* 1858 * If iconv will take the string, do not change the 1859 * setting of the normalization. 1860 */ 1861 if (!(sc->flag & SCONV_WIN_CP) && 1862 (sc->flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8)) && 1863 !(sc->flag & (SCONV_TO_UTF16 | SCONV_TO_UTF8))) 1864 break; 1865 #endif 1866 if ((sc->flag & SCONV_NORMALIZATION_D) == 0) { 1867 sc->flag |= SCONV_NORMALIZATION_D; 1868 sc->flag &= ~SCONV_NORMALIZATION_C; 1869 /* Set up string converters. */ 1870 setup_converter(sc); 1871 } 1872 break; 1873 default: 1874 break; 1875 } 1876 } 1877 1878 /* 1879 * 1880 * Copy one archive_string to another in locale conversion. 1881 * 1882 * archive_strncat_l(); 1883 * archive_strncpy_l(); 1884 * 1885 */ 1886 1887 static size_t 1888 mbsnbytes(const void *_p, size_t n) 1889 { 1890 size_t s; 1891 const char *p, *pp; 1892 1893 if (_p == NULL) 1894 return (0); 1895 p = (const char *)_p; 1896 1897 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1898 s = 0; 1899 pp = p; 1900 while (s < n && *pp) { 1901 pp++; 1902 s++; 1903 } 1904 return (s); 1905 } 1906 1907 static size_t 1908 utf16nbytes(const void *_p, size_t n) 1909 { 1910 size_t s; 1911 const char *p, *pp; 1912 1913 if (_p == NULL) 1914 return (0); 1915 p = (const char *)_p; 1916 1917 /* Like strlen(p), except won't examine positions beyond p[n]. */ 1918 s = 0; 1919 pp = p; 1920 n >>= 1; 1921 while (s < n && (pp[0] || pp[1])) { 1922 pp += 2; 1923 s++; 1924 } 1925 return (s<<1); 1926 } 1927 1928 int 1929 archive_strncpy_l(struct archive_string *as, const void *_p, size_t n, 1930 struct archive_string_conv *sc) 1931 { 1932 as->length = 0; 1933 return (archive_strncat_l(as, _p, n, sc)); 1934 } 1935 1936 int 1937 archive_strncat_l(struct archive_string *as, const void *_p, size_t n, 1938 struct archive_string_conv *sc) 1939 { 1940 const void *s; 1941 size_t length; 1942 int i, r = 0, r2; 1943 1944 /* We must allocate memory even if there is no data for conversion 1945 * or copy. This simulates archive_string_append behavior. */ 1946 if (_p == NULL || n == 0) { 1947 int tn = 1; 1948 if (sc != NULL && (sc->flag & SCONV_TO_UTF16)) 1949 tn = 2; 1950 if (archive_string_ensure(as, as->length + tn) == NULL) 1951 return (-1); 1952 as->s[as->length] = 0; 1953 if (tn == 2) 1954 as->s[as->length+1] = 0; 1955 return (0); 1956 } 1957 1958 /* 1959 * If sc is NULL, we just make a copy. 1960 */ 1961 if (sc == NULL) { 1962 length = mbsnbytes(_p, n); 1963 if (archive_string_append(as, _p, length) == NULL) 1964 return (-1);/* No memory */ 1965 return (0); 1966 } 1967 1968 if (sc->flag & SCONV_FROM_UTF16) 1969 length = utf16nbytes(_p, n); 1970 else 1971 length = mbsnbytes(_p, n); 1972 s = _p; 1973 i = 0; 1974 if (sc->nconverter > 1) { 1975 sc->utftmp.length = 0; 1976 r2 = sc->converter[0](&(sc->utftmp), s, length, sc); 1977 if (r2 != 0 && errno == ENOMEM) 1978 return (r2); 1979 if (r > r2) 1980 r = r2; 1981 s = sc->utftmp.s; 1982 length = sc->utftmp.length; 1983 ++i; 1984 } 1985 r2 = sc->converter[i](as, s, length, sc); 1986 if (r > r2) 1987 r = r2; 1988 return (r); 1989 } 1990 1991 #if HAVE_ICONV 1992 1993 /* 1994 * Return -1 if conversion failes. 1995 */ 1996 static int 1997 iconv_strncat_in_locale(struct archive_string *as, const void *_p, 1998 size_t length, struct archive_string_conv *sc) 1999 { 2000 ICONV_CONST char *itp; 2001 size_t remaining; 2002 iconv_t cd; 2003 char *outp; 2004 size_t avail, bs; 2005 int return_value = 0; /* success */ 2006 int to_size, from_size; 2007 2008 if (sc->flag & SCONV_TO_UTF16) 2009 to_size = 2; 2010 else 2011 to_size = 1; 2012 if (sc->flag & SCONV_FROM_UTF16) 2013 from_size = 2; 2014 else 2015 from_size = 1; 2016 2017 if (archive_string_ensure(as, as->length + length*2+to_size) == NULL) 2018 return (-1); 2019 2020 cd = sc->cd; 2021 itp = (char *)(uintptr_t)_p; 2022 remaining = length; 2023 outp = as->s + as->length; 2024 avail = as->buffer_length - as->length - to_size; 2025 while (remaining >= (size_t)from_size) { 2026 size_t result = iconv(cd, &itp, &remaining, &outp, &avail); 2027 2028 if (result != (size_t)-1) 2029 break; /* Conversion completed. */ 2030 2031 if (errno == EILSEQ || errno == EINVAL) { 2032 /* 2033 * If an output charset is UTF-8 or UTF-16BE/LE, 2034 * unknown character should be U+FFFD 2035 * (replacement character). 2036 */ 2037 if (sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) { 2038 size_t rbytes; 2039 if (sc->flag & SCONV_TO_UTF8) 2040 rbytes = sizeof(utf8_replacement_char); 2041 else 2042 rbytes = 2; 2043 2044 if (avail < rbytes) { 2045 as->length = outp - as->s; 2046 bs = as->buffer_length + 2047 (remaining * to_size) + rbytes; 2048 if (NULL == 2049 archive_string_ensure(as, bs)) 2050 return (-1); 2051 outp = as->s + as->length; 2052 avail = as->buffer_length 2053 - as->length - to_size; 2054 } 2055 if (sc->flag & SCONV_TO_UTF8) 2056 memcpy(outp, utf8_replacement_char, sizeof(utf8_replacement_char)); 2057 else if (sc->flag & SCONV_TO_UTF16BE) 2058 archive_be16enc(outp, UNICODE_R_CHAR); 2059 else 2060 archive_le16enc(outp, UNICODE_R_CHAR); 2061 outp += rbytes; 2062 avail -= rbytes; 2063 } else { 2064 /* Skip the illegal input bytes. */ 2065 *outp++ = '?'; 2066 avail--; 2067 } 2068 itp += from_size; 2069 remaining -= from_size; 2070 return_value = -1; /* failure */ 2071 } else { 2072 /* E2BIG no output buffer, 2073 * Increase an output buffer. */ 2074 as->length = outp - as->s; 2075 bs = as->buffer_length + remaining * 2; 2076 if (NULL == archive_string_ensure(as, bs)) 2077 return (-1); 2078 outp = as->s + as->length; 2079 avail = as->buffer_length - as->length - to_size; 2080 } 2081 } 2082 as->length = outp - as->s; 2083 as->s[as->length] = 0; 2084 if (to_size == 2) 2085 as->s[as->length+1] = 0; 2086 return (return_value); 2087 } 2088 2089 #endif /* HAVE_ICONV */ 2090 2091 2092 #if defined(_WIN32) && !defined(__CYGWIN__) 2093 2094 /* 2095 * Translate a string from a some CodePage to an another CodePage by 2096 * Windows APIs, and copy the result. Return -1 if conversion failes. 2097 */ 2098 static int 2099 strncat_in_codepage(struct archive_string *as, 2100 const void *_p, size_t length, struct archive_string_conv *sc) 2101 { 2102 const char *s = (const char *)_p; 2103 struct archive_wstring aws; 2104 size_t l; 2105 int r, saved_flag; 2106 2107 archive_string_init(&aws); 2108 saved_flag = sc->flag; 2109 sc->flag &= ~(SCONV_NORMALIZATION_D | SCONV_NORMALIZATION_C); 2110 r = archive_wstring_append_from_mbs_in_codepage(&aws, s, length, sc); 2111 sc->flag = saved_flag; 2112 if (r != 0) { 2113 archive_wstring_free(&aws); 2114 if (errno != ENOMEM) 2115 archive_string_append(as, s, length); 2116 return (-1); 2117 } 2118 2119 l = as->length; 2120 r = archive_string_append_from_wcs_in_codepage( 2121 as, aws.s, aws.length, sc); 2122 if (r != 0 && errno != ENOMEM && l == as->length) 2123 archive_string_append(as, s, length); 2124 archive_wstring_free(&aws); 2125 return (r); 2126 } 2127 2128 /* 2129 * Test whether MBS ==> WCS is okay. 2130 */ 2131 static int 2132 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2133 { 2134 const char *p = (const char *)_p; 2135 unsigned codepage; 2136 DWORD mbflag = MB_ERR_INVALID_CHARS; 2137 2138 if (sc->flag & SCONV_FROM_CHARSET) 2139 codepage = sc->to_cp; 2140 else 2141 codepage = sc->from_cp; 2142 2143 if (codepage == CP_C_LOCALE) 2144 return (0); 2145 if (codepage != CP_UTF8) 2146 mbflag |= MB_PRECOMPOSED; 2147 2148 if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0) 2149 return (-1); /* Invalid */ 2150 return (0); /* Okay */ 2151 } 2152 2153 #else 2154 2155 /* 2156 * Test whether MBS ==> WCS is okay. 2157 */ 2158 static int 2159 invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc) 2160 { 2161 const char *p = (const char *)_p; 2162 size_t r; 2163 2164 #if HAVE_MBRTOWC 2165 mbstate_t shift_state; 2166 2167 memset(&shift_state, 0, sizeof(shift_state)); 2168 #else 2169 /* Clear the shift state before starting. */ 2170 mbtowc(NULL, NULL, 0); 2171 #endif 2172 while (n) { 2173 wchar_t wc; 2174 2175 #if HAVE_MBRTOWC 2176 r = mbrtowc(&wc, p, n, &shift_state); 2177 #else 2178 r = mbtowc(&wc, p, n); 2179 #endif 2180 if (r == (size_t)-1 || r == (size_t)-2) 2181 return (-1);/* Invalid. */ 2182 if (r == 0) 2183 break; 2184 p += r; 2185 n -= r; 2186 } 2187 (void)sc; /* UNUSED */ 2188 return (0); /* All Okey. */ 2189 } 2190 2191 #endif /* defined(_WIN32) && !defined(__CYGWIN__) */ 2192 2193 /* 2194 * Basically returns -1 because we cannot make a conversion of charset 2195 * without iconv but in some cases this would return 0. 2196 * Returns 0 if all copied characters are ASCII. 2197 * Returns 0 if both from-locale and to-locale are the same and those 2198 * can be WCS with no error. 2199 */ 2200 static int 2201 best_effort_strncat_in_locale(struct archive_string *as, const void *_p, 2202 size_t length, struct archive_string_conv *sc) 2203 { 2204 size_t remaining; 2205 const uint8_t *itp; 2206 int return_value = 0; /* success */ 2207 2208 /* 2209 * If both from-locale and to-locale is the same, this makes a copy. 2210 * And then this checks all copied MBS can be WCS if so returns 0. 2211 */ 2212 if (sc->same) { 2213 if (archive_string_append(as, _p, length) == NULL) 2214 return (-1);/* No memory */ 2215 return (invalid_mbs(_p, length, sc)); 2216 } 2217 2218 /* 2219 * If a character is ASCII, this just copies it. If not, this 2220 * assigns '?' charater instead but in UTF-8 locale this assigns 2221 * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD, 2222 * a Replacement Character in Unicode. 2223 */ 2224 2225 remaining = length; 2226 itp = (const uint8_t *)_p; 2227 while (*itp && remaining > 0) { 2228 if (*itp > 127) { 2229 // Non-ASCII: Substitute with suitable replacement 2230 if (sc->flag & SCONV_TO_UTF8) { 2231 if (archive_string_append(as, utf8_replacement_char, sizeof(utf8_replacement_char)) == NULL) { 2232 __archive_errx(1, "Out of memory"); 2233 } 2234 } else { 2235 archive_strappend_char(as, '?'); 2236 } 2237 return_value = -1; 2238 } else { 2239 archive_strappend_char(as, *itp); 2240 } 2241 ++itp; 2242 } 2243 return (return_value); 2244 } 2245 2246 2247 /* 2248 * Unicode conversion functions. 2249 * - UTF-8 <===> UTF-8 in removing surrogate pairs. 2250 * - UTF-8 NFD ===> UTF-8 NFC in removing surrogate pairs. 2251 * - UTF-8 made by libarchive 2.x ===> UTF-8. 2252 * - UTF-16BE <===> UTF-8. 2253 * 2254 */ 2255 2256 /* 2257 * Utility to convert a single UTF-8 sequence. 2258 * 2259 * Usually return used bytes, return used byte in negative value when 2260 * a unicode character is replaced with U+FFFD. 2261 * See also http://unicode.org/review/pr-121.html Public Review Issue #121 2262 * Recommended Practice for Replacement Characters. 2263 */ 2264 static int 2265 _utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2266 { 2267 static const char utf8_count[256] = { 2268 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 00 - 0F */ 2269 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 10 - 1F */ 2270 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 20 - 2F */ 2271 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 30 - 3F */ 2272 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 40 - 4F */ 2273 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 50 - 5F */ 2274 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 60 - 6F */ 2275 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,/* 70 - 7F */ 2276 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 80 - 8F */ 2277 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* 90 - 9F */ 2278 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* A0 - AF */ 2279 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,/* B0 - BF */ 2280 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* C0 - CF */ 2281 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,/* D0 - DF */ 2282 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,/* E0 - EF */ 2283 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0 - FF */ 2284 }; 2285 int ch, i; 2286 int cnt; 2287 uint32_t wc; 2288 2289 /* Sanity check. */ 2290 if (n == 0) 2291 return (0); 2292 /* 2293 * Decode 1-4 bytes depending on the value of the first byte. 2294 */ 2295 ch = (unsigned char)*s; 2296 if (ch == 0) 2297 return (0); /* Standard: return 0 for end-of-string. */ 2298 cnt = utf8_count[ch]; 2299 2300 /* Invalide sequence or there are not plenty bytes. */ 2301 if ((int)n < cnt) { 2302 cnt = (int)n; 2303 for (i = 1; i < cnt; i++) { 2304 if ((s[i] & 0xc0) != 0x80) { 2305 cnt = i; 2306 break; 2307 } 2308 } 2309 goto invalid_sequence; 2310 } 2311 2312 /* Make a Unicode code point from a single UTF-8 sequence. */ 2313 switch (cnt) { 2314 case 1: /* 1 byte sequence. */ 2315 *pwc = ch & 0x7f; 2316 return (cnt); 2317 case 2: /* 2 bytes sequence. */ 2318 if ((s[1] & 0xc0) != 0x80) { 2319 cnt = 1; 2320 goto invalid_sequence; 2321 } 2322 *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f); 2323 return (cnt); 2324 case 3: /* 3 bytes sequence. */ 2325 if ((s[1] & 0xc0) != 0x80) { 2326 cnt = 1; 2327 goto invalid_sequence; 2328 } 2329 if ((s[2] & 0xc0) != 0x80) { 2330 cnt = 2; 2331 goto invalid_sequence; 2332 } 2333 wc = ((ch & 0x0f) << 12) 2334 | ((s[1] & 0x3f) << 6) 2335 | (s[2] & 0x3f); 2336 if (wc < 0x800) 2337 goto invalid_sequence;/* Overlong sequence. */ 2338 break; 2339 case 4: /* 4 bytes sequence. */ 2340 if ((s[1] & 0xc0) != 0x80) { 2341 cnt = 1; 2342 goto invalid_sequence; 2343 } 2344 if ((s[2] & 0xc0) != 0x80) { 2345 cnt = 2; 2346 goto invalid_sequence; 2347 } 2348 if ((s[3] & 0xc0) != 0x80) { 2349 cnt = 3; 2350 goto invalid_sequence; 2351 } 2352 wc = ((ch & 0x07) << 18) 2353 | ((s[1] & 0x3f) << 12) 2354 | ((s[2] & 0x3f) << 6) 2355 | (s[3] & 0x3f); 2356 if (wc < 0x10000) 2357 goto invalid_sequence;/* Overlong sequence. */ 2358 break; 2359 default: /* Others are all invalid sequence. */ 2360 if (ch == 0xc0 || ch == 0xc1) 2361 cnt = 2; 2362 else if (ch >= 0xf5 && ch <= 0xf7) 2363 cnt = 4; 2364 else if (ch >= 0xf8 && ch <= 0xfb) 2365 cnt = 5; 2366 else if (ch == 0xfc || ch == 0xfd) 2367 cnt = 6; 2368 else 2369 cnt = 1; 2370 if ((int)n < cnt) 2371 cnt = (int)n; 2372 for (i = 1; i < cnt; i++) { 2373 if ((s[i] & 0xc0) != 0x80) { 2374 cnt = i; 2375 break; 2376 } 2377 } 2378 goto invalid_sequence; 2379 } 2380 2381 /* The code point larger than 0x10FFFF is not leagal 2382 * Unicode values. */ 2383 if (wc > UNICODE_MAX) 2384 goto invalid_sequence; 2385 /* Correctly gets a Unicode, returns used bytes. */ 2386 *pwc = wc; 2387 return (cnt); 2388 invalid_sequence: 2389 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2390 return (cnt * -1); 2391 } 2392 2393 static int 2394 utf8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2395 { 2396 int cnt; 2397 2398 cnt = _utf8_to_unicode(pwc, s, n); 2399 /* Any of Surrogate pair is not leagal Unicode values. */ 2400 if (cnt == 3 && IS_SURROGATE_PAIR_LA(*pwc)) 2401 return (-3); 2402 return (cnt); 2403 } 2404 2405 static inline uint32_t 2406 combine_surrogate_pair(uint32_t uc, uint32_t uc2) 2407 { 2408 uc -= 0xD800; 2409 uc *= 0x400; 2410 uc += uc2 - 0xDC00; 2411 uc += 0x10000; 2412 return (uc); 2413 } 2414 2415 /* 2416 * Convert a single UTF-8/CESU-8 sequence to a Unicode code point in 2417 * removing surrogate pairs. 2418 * 2419 * CESU-8: The Compatibility Encoding Scheme for UTF-16. 2420 * 2421 * Usually return used bytes, return used byte in negative value when 2422 * a unicode character is replaced with U+FFFD. 2423 */ 2424 static int 2425 cesu8_to_unicode(uint32_t *pwc, const char *s, size_t n) 2426 { 2427 uint32_t wc = 0; 2428 int cnt; 2429 2430 cnt = _utf8_to_unicode(&wc, s, n); 2431 if (cnt == 3 && IS_HIGH_SURROGATE_LA(wc)) { 2432 uint32_t wc2 = 0; 2433 if (n - 3 < 3) { 2434 /* Invalid byte sequence. */ 2435 goto invalid_sequence; 2436 } 2437 cnt = _utf8_to_unicode(&wc2, s+3, n-3); 2438 if (cnt != 3 || !IS_LOW_SURROGATE_LA(wc2)) { 2439 /* Invalid byte sequence. */ 2440 goto invalid_sequence; 2441 } 2442 wc = combine_surrogate_pair(wc, wc2); 2443 cnt = 6; 2444 } else if (cnt == 3 && IS_LOW_SURROGATE_LA(wc)) { 2445 /* Invalid byte sequence. */ 2446 goto invalid_sequence; 2447 } 2448 *pwc = wc; 2449 return (cnt); 2450 invalid_sequence: 2451 *pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */ 2452 if (cnt > 0) 2453 cnt *= -1; 2454 return (cnt); 2455 } 2456 2457 /* 2458 * Convert a Unicode code point to a single UTF-8 sequence. 2459 * 2460 * NOTE:This function does not check if the Unicode is leagal or not. 2461 * Please you definitely check it before calling this. 2462 */ 2463 static size_t 2464 unicode_to_utf8(char *p, size_t remaining, uint32_t uc) 2465 { 2466 char *_p = p; 2467 2468 /* Invalid Unicode char maps to Replacement character */ 2469 if (uc > UNICODE_MAX) 2470 uc = UNICODE_R_CHAR; 2471 /* Translate code point to UTF8 */ 2472 if (uc <= 0x7f) { 2473 if (remaining == 0) 2474 return (0); 2475 *p++ = (char)uc; 2476 } else if (uc <= 0x7ff) { 2477 if (remaining < 2) 2478 return (0); 2479 *p++ = 0xc0 | ((uc >> 6) & 0x1f); 2480 *p++ = 0x80 | (uc & 0x3f); 2481 } else if (uc <= 0xffff) { 2482 if (remaining < 3) 2483 return (0); 2484 *p++ = 0xe0 | ((uc >> 12) & 0x0f); 2485 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2486 *p++ = 0x80 | (uc & 0x3f); 2487 } else { 2488 if (remaining < 4) 2489 return (0); 2490 *p++ = 0xf0 | ((uc >> 18) & 0x07); 2491 *p++ = 0x80 | ((uc >> 12) & 0x3f); 2492 *p++ = 0x80 | ((uc >> 6) & 0x3f); 2493 *p++ = 0x80 | (uc & 0x3f); 2494 } 2495 return (p - _p); 2496 } 2497 2498 static int 2499 utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n) 2500 { 2501 return (utf16_to_unicode(pwc, s, n, 1)); 2502 } 2503 2504 static int 2505 utf16le_to_unicode(uint32_t *pwc, const char *s, size_t n) 2506 { 2507 return (utf16_to_unicode(pwc, s, n, 0)); 2508 } 2509 2510 static int 2511 utf16_to_unicode(uint32_t *pwc, const char *s, size_t n, int be) 2512 { 2513 const char *utf16 = s; 2514 unsigned uc; 2515 2516 if (n == 0) 2517 return (0); 2518 if (n == 1) { 2519 /* set the Replacement Character instead. */ 2520 *pwc = UNICODE_R_CHAR; 2521 return (-1); 2522 } 2523 2524 if (be) 2525 uc = archive_be16dec(utf16); 2526 else 2527 uc = archive_le16dec(utf16); 2528 utf16 += 2; 2529 2530 /* If this is a surrogate pair, assemble the full code point.*/ 2531 if (IS_HIGH_SURROGATE_LA(uc)) { 2532 unsigned uc2; 2533 2534 if (n >= 4) { 2535 if (be) 2536 uc2 = archive_be16dec(utf16); 2537 else 2538 uc2 = archive_le16dec(utf16); 2539 } else 2540 uc2 = 0; 2541 if (IS_LOW_SURROGATE_LA(uc2)) { 2542 uc = combine_surrogate_pair(uc, uc2); 2543 utf16 += 2; 2544 } else { 2545 /* Undescribed code point should be U+FFFD 2546 * (replacement character). */ 2547 *pwc = UNICODE_R_CHAR; 2548 return (-2); 2549 } 2550 } 2551 2552 /* 2553 * Surrogate pair values(0xd800 through 0xdfff) are only 2554 * used by UTF-16, so, after above culculation, the code 2555 * must not be surrogate values, and Unicode has no codes 2556 * larger than 0x10ffff. Thus, those are not leagal Unicode 2557 * values. 2558 */ 2559 if (IS_SURROGATE_PAIR_LA(uc) || uc > UNICODE_MAX) { 2560 /* Undescribed code point should be U+FFFD 2561 * (replacement character). */ 2562 *pwc = UNICODE_R_CHAR; 2563 return (((int)(utf16 - s)) * -1); 2564 } 2565 *pwc = uc; 2566 return ((int)(utf16 - s)); 2567 } 2568 2569 static size_t 2570 unicode_to_utf16be(char *p, size_t remaining, uint32_t uc) 2571 { 2572 char *utf16 = p; 2573 2574 if (uc > 0xffff) { 2575 /* We have a code point that won't fit into a 2576 * wchar_t; convert it to a surrogate pair. */ 2577 if (remaining < 4) 2578 return (0); 2579 uc -= 0x10000; 2580 archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2581 archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2582 return (4); 2583 } else { 2584 if (remaining < 2) 2585 return (0); 2586 archive_be16enc(utf16, uc); 2587 return (2); 2588 } 2589 } 2590 2591 static size_t 2592 unicode_to_utf16le(char *p, size_t remaining, uint32_t uc) 2593 { 2594 char *utf16 = p; 2595 2596 if (uc > 0xffff) { 2597 /* We have a code point that won't fit into a 2598 * wchar_t; convert it to a surrogate pair. */ 2599 if (remaining < 4) 2600 return (0); 2601 uc -= 0x10000; 2602 archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800); 2603 archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00); 2604 return (4); 2605 } else { 2606 if (remaining < 2) 2607 return (0); 2608 archive_le16enc(utf16, uc); 2609 return (2); 2610 } 2611 } 2612 2613 /* 2614 * Copy UTF-8 string in checking surrogate pair. 2615 * If any surrogate pair are found, it would be canonicalized. 2616 */ 2617 static int 2618 strncat_from_utf8_to_utf8(struct archive_string *as, const void *_p, 2619 size_t len, struct archive_string_conv *sc) 2620 { 2621 const char *s; 2622 char *p, *endp; 2623 int n, ret = 0; 2624 2625 (void)sc; /* UNUSED */ 2626 2627 if (archive_string_ensure(as, as->length + len + 1) == NULL) 2628 return (-1); 2629 2630 s = (const char *)_p; 2631 p = as->s + as->length; 2632 endp = as->s + as->buffer_length -1; 2633 do { 2634 uint32_t uc; 2635 const char *ss = s; 2636 size_t w; 2637 2638 /* 2639 * Forward byte sequence until a conversion of that is needed. 2640 */ 2641 while ((n = utf8_to_unicode(&uc, s, len)) > 0) { 2642 s += n; 2643 len -= n; 2644 } 2645 if (ss < s) { 2646 if (p + (s - ss) > endp) { 2647 as->length = p - as->s; 2648 if (archive_string_ensure(as, 2649 as->buffer_length + len + 1) == NULL) 2650 return (-1); 2651 p = as->s + as->length; 2652 endp = as->s + as->buffer_length -1; 2653 } 2654 2655 memcpy(p, ss, s - ss); 2656 p += s - ss; 2657 } 2658 2659 /* 2660 * If n is negative, current byte sequence needs a replacement. 2661 */ 2662 if (n < 0) { 2663 if (n == -3 && IS_SURROGATE_PAIR_LA(uc)) { 2664 /* Current byte sequence may be CESU-8. */ 2665 n = cesu8_to_unicode(&uc, s, len); 2666 } 2667 if (n < 0) { 2668 ret = -1; 2669 n *= -1;/* Use a replaced unicode character. */ 2670 } 2671 2672 /* Rebuild UTF-8 byte sequence. */ 2673 while ((w = unicode_to_utf8(p, endp - p, uc)) == 0) { 2674 as->length = p - as->s; 2675 if (archive_string_ensure(as, 2676 as->buffer_length + len + 1) == NULL) 2677 return (-1); 2678 p = as->s + as->length; 2679 endp = as->s + as->buffer_length -1; 2680 } 2681 p += w; 2682 s += n; 2683 len -= n; 2684 } 2685 } while (n > 0); 2686 as->length = p - as->s; 2687 as->s[as->length] = '\0'; 2688 return (ret); 2689 } 2690 2691 static int 2692 archive_string_append_unicode(struct archive_string *as, const void *_p, 2693 size_t len, struct archive_string_conv *sc) 2694 { 2695 const char *s; 2696 char *p, *endp; 2697 uint32_t uc; 2698 size_t w; 2699 int n, ret = 0, ts, tm; 2700 int (*parse)(uint32_t *, const char *, size_t); 2701 size_t (*unparse)(char *, size_t, uint32_t); 2702 2703 if (sc->flag & SCONV_TO_UTF16BE) { 2704 unparse = unicode_to_utf16be; 2705 ts = 2; 2706 } else if (sc->flag & SCONV_TO_UTF16LE) { 2707 unparse = unicode_to_utf16le; 2708 ts = 2; 2709 } else if (sc->flag & SCONV_TO_UTF8) { 2710 unparse = unicode_to_utf8; 2711 ts = 1; 2712 } else { 2713 /* 2714 * This case is going to be converted to another 2715 * character-set through iconv. 2716 */ 2717 if (sc->flag & SCONV_FROM_UTF16BE) { 2718 unparse = unicode_to_utf16be; 2719 ts = 2; 2720 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2721 unparse = unicode_to_utf16le; 2722 ts = 2; 2723 } else { 2724 unparse = unicode_to_utf8; 2725 ts = 1; 2726 } 2727 } 2728 2729 if (sc->flag & SCONV_FROM_UTF16BE) { 2730 parse = utf16be_to_unicode; 2731 tm = 1; 2732 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2733 parse = utf16le_to_unicode; 2734 tm = 1; 2735 } else { 2736 parse = cesu8_to_unicode; 2737 tm = ts; 2738 } 2739 2740 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2741 return (-1); 2742 2743 s = (const char *)_p; 2744 p = as->s + as->length; 2745 endp = as->s + as->buffer_length - ts; 2746 while ((n = parse(&uc, s, len)) != 0) { 2747 if (n < 0) { 2748 /* Use a replaced unicode character. */ 2749 n *= -1; 2750 ret = -1; 2751 } 2752 s += n; 2753 len -= n; 2754 while ((w = unparse(p, endp - p, uc)) == 0) { 2755 /* There is not enough output buffer so 2756 * we have to expand it. */ 2757 as->length = p - as->s; 2758 if (archive_string_ensure(as, 2759 as->buffer_length + len * tm + ts) == NULL) 2760 return (-1); 2761 p = as->s + as->length; 2762 endp = as->s + as->buffer_length - ts; 2763 } 2764 p += w; 2765 } 2766 as->length = p - as->s; 2767 as->s[as->length] = '\0'; 2768 if (ts == 2) 2769 as->s[as->length+1] = '\0'; 2770 return (ret); 2771 } 2772 2773 /* 2774 * Following Constants for Hangul compositions this information comes from 2775 * Unicode Standard Annex #15 http://unicode.org/reports/tr15/ 2776 */ 2777 #define HC_SBASE 0xAC00 2778 #define HC_LBASE 0x1100 2779 #define HC_VBASE 0x1161 2780 #define HC_TBASE 0x11A7 2781 #define HC_LCOUNT 19 2782 #define HC_VCOUNT 21 2783 #define HC_TCOUNT 28 2784 #define HC_NCOUNT (HC_VCOUNT * HC_TCOUNT) 2785 #define HC_SCOUNT (HC_LCOUNT * HC_NCOUNT) 2786 2787 static uint32_t 2788 get_nfc(uint32_t uc, uint32_t uc2) 2789 { 2790 int t, b; 2791 2792 t = 0; 2793 b = sizeof(u_composition_table)/sizeof(u_composition_table[0]) -1; 2794 while (b >= t) { 2795 int m = (t + b) / 2; 2796 if (u_composition_table[m].cp1 < uc) 2797 t = m + 1; 2798 else if (u_composition_table[m].cp1 > uc) 2799 b = m - 1; 2800 else if (u_composition_table[m].cp2 < uc2) 2801 t = m + 1; 2802 else if (u_composition_table[m].cp2 > uc2) 2803 b = m - 1; 2804 else 2805 return (u_composition_table[m].nfc); 2806 } 2807 return (0); 2808 } 2809 2810 #define FDC_MAX 10 /* The maximum number of Following Decomposable 2811 * Characters. */ 2812 2813 /* 2814 * Update first code point. 2815 */ 2816 #define UPDATE_UC(new_uc) do { \ 2817 uc = new_uc; \ 2818 ucptr = NULL; \ 2819 } while (0) 2820 2821 /* 2822 * Replace first code point with second code point. 2823 */ 2824 #define REPLACE_UC_WITH_UC2() do { \ 2825 uc = uc2; \ 2826 ucptr = uc2ptr; \ 2827 n = n2; \ 2828 } while (0) 2829 2830 #define EXPAND_BUFFER() do { \ 2831 as->length = p - as->s; \ 2832 if (archive_string_ensure(as, \ 2833 as->buffer_length + len * tm + ts) == NULL)\ 2834 return (-1); \ 2835 p = as->s + as->length; \ 2836 endp = as->s + as->buffer_length - ts; \ 2837 } while (0) 2838 2839 #define UNPARSE(p, endp, uc) do { \ 2840 while ((w = unparse(p, (endp) - (p), uc)) == 0) {\ 2841 EXPAND_BUFFER(); \ 2842 } \ 2843 p += w; \ 2844 } while (0) 2845 2846 /* 2847 * Write first code point. 2848 * If the code point has not be changed from its original code, 2849 * this just copies it from its original buffer pointer. 2850 * If not, this converts it to UTF-8 byte sequence and copies it. 2851 */ 2852 #define WRITE_UC() do { \ 2853 if (ucptr) { \ 2854 if (p + n > endp) \ 2855 EXPAND_BUFFER(); \ 2856 switch (n) { \ 2857 case 4: \ 2858 *p++ = *ucptr++; \ 2859 /* FALL THROUGH */ \ 2860 case 3: \ 2861 *p++ = *ucptr++; \ 2862 /* FALL THROUGH */ \ 2863 case 2: \ 2864 *p++ = *ucptr++; \ 2865 /* FALL THROUGH */ \ 2866 case 1: \ 2867 *p++ = *ucptr; \ 2868 break; \ 2869 } \ 2870 ucptr = NULL; \ 2871 } else { \ 2872 UNPARSE(p, endp, uc); \ 2873 } \ 2874 } while (0) 2875 2876 /* 2877 * Collect following decomposable code points. 2878 */ 2879 #define COLLECT_CPS(start) do { \ 2880 int _i; \ 2881 for (_i = start; _i < FDC_MAX ; _i++) { \ 2882 nx = parse(&ucx[_i], s, len); \ 2883 if (nx <= 0) \ 2884 break; \ 2885 cx = CCC(ucx[_i]); \ 2886 if (cl >= cx && cl != 228 && cx != 228)\ 2887 break; \ 2888 s += nx; \ 2889 len -= nx; \ 2890 cl = cx; \ 2891 ccx[_i] = cx; \ 2892 } \ 2893 if (_i >= FDC_MAX) { \ 2894 ret = -1; \ 2895 ucx_size = FDC_MAX; \ 2896 } else \ 2897 ucx_size = _i; \ 2898 } while (0) 2899 2900 /* 2901 * Normalize UTF-8/UTF-16BE characters to Form C and copy the result. 2902 * 2903 * TODO: Convert composition exclusions,which are never converted 2904 * from NFC,NFD,NFKC and NFKD, to Form C. 2905 */ 2906 static int 2907 archive_string_normalize_C(struct archive_string *as, const void *_p, 2908 size_t len, struct archive_string_conv *sc) 2909 { 2910 const char *s = (const char *)_p; 2911 char *p, *endp; 2912 uint32_t uc, uc2; 2913 size_t w; 2914 int always_replace, n, n2, ret = 0, spair, ts, tm; 2915 int (*parse)(uint32_t *, const char *, size_t); 2916 size_t (*unparse)(char *, size_t, uint32_t); 2917 2918 always_replace = 1; 2919 ts = 1;/* text size. */ 2920 if (sc->flag & SCONV_TO_UTF16BE) { 2921 unparse = unicode_to_utf16be; 2922 ts = 2; 2923 if (sc->flag & SCONV_FROM_UTF16BE) 2924 always_replace = 0; 2925 } else if (sc->flag & SCONV_TO_UTF16LE) { 2926 unparse = unicode_to_utf16le; 2927 ts = 2; 2928 if (sc->flag & SCONV_FROM_UTF16LE) 2929 always_replace = 0; 2930 } else if (sc->flag & SCONV_TO_UTF8) { 2931 unparse = unicode_to_utf8; 2932 if (sc->flag & SCONV_FROM_UTF8) 2933 always_replace = 0; 2934 } else { 2935 /* 2936 * This case is going to be converted to another 2937 * character-set through iconv. 2938 */ 2939 always_replace = 0; 2940 if (sc->flag & SCONV_FROM_UTF16BE) { 2941 unparse = unicode_to_utf16be; 2942 ts = 2; 2943 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2944 unparse = unicode_to_utf16le; 2945 ts = 2; 2946 } else { 2947 unparse = unicode_to_utf8; 2948 } 2949 } 2950 2951 if (sc->flag & SCONV_FROM_UTF16BE) { 2952 parse = utf16be_to_unicode; 2953 tm = 1; 2954 spair = 4;/* surrogate pair size in UTF-16. */ 2955 } else if (sc->flag & SCONV_FROM_UTF16LE) { 2956 parse = utf16le_to_unicode; 2957 tm = 1; 2958 spair = 4;/* surrogate pair size in UTF-16. */ 2959 } else { 2960 parse = cesu8_to_unicode; 2961 tm = ts; 2962 spair = 6;/* surrogate pair size in UTF-8. */ 2963 } 2964 2965 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 2966 return (-1); 2967 2968 p = as->s + as->length; 2969 endp = as->s + as->buffer_length - ts; 2970 while ((n = parse(&uc, s, len)) != 0) { 2971 const char *ucptr, *uc2ptr; 2972 2973 if (n < 0) { 2974 /* Use a replaced unicode character. */ 2975 UNPARSE(p, endp, uc); 2976 s += n*-1; 2977 len -= n*-1; 2978 ret = -1; 2979 continue; 2980 } else if (n == spair || always_replace) 2981 /* uc is converted from a surrogate pair. 2982 * this should be treated as a changed code. */ 2983 ucptr = NULL; 2984 else 2985 ucptr = s; 2986 s += n; 2987 len -= n; 2988 2989 /* Read second code point. */ 2990 while ((n2 = parse(&uc2, s, len)) > 0) { 2991 uint32_t ucx[FDC_MAX]; 2992 int ccx[FDC_MAX]; 2993 int cl, cx, i, nx, ucx_size; 2994 int LIndex,SIndex; 2995 uint32_t nfc; 2996 2997 if (n2 == spair || always_replace) 2998 /* uc2 is converted from a surrogate pair. 2999 * this should be treated as a changed code. */ 3000 uc2ptr = NULL; 3001 else 3002 uc2ptr = s; 3003 s += n2; 3004 len -= n2; 3005 3006 /* 3007 * If current second code point is out of decomposable 3008 * code points, finding compositions is unneeded. 3009 */ 3010 if (!IS_DECOMPOSABLE_BLOCK(uc2)) { 3011 WRITE_UC(); 3012 REPLACE_UC_WITH_UC2(); 3013 continue; 3014 } 3015 3016 /* 3017 * Try to combine current code points. 3018 */ 3019 /* 3020 * We have to combine Hangul characters according to 3021 * http://uniicode.org/reports/tr15/#Hangul 3022 */ 3023 if (0 <= (LIndex = uc - HC_LBASE) && 3024 LIndex < HC_LCOUNT) { 3025 /* 3026 * Hangul Composition. 3027 * 1. Two current code points are L and V. 3028 */ 3029 int VIndex = uc2 - HC_VBASE; 3030 if (0 <= VIndex && VIndex < HC_VCOUNT) { 3031 /* Make syllable of form LV. */ 3032 UPDATE_UC(HC_SBASE + 3033 (LIndex * HC_VCOUNT + VIndex) * 3034 HC_TCOUNT); 3035 } else { 3036 WRITE_UC(); 3037 REPLACE_UC_WITH_UC2(); 3038 } 3039 continue; 3040 } else if (0 <= (SIndex = uc - HC_SBASE) && 3041 SIndex < HC_SCOUNT && (SIndex % HC_TCOUNT) == 0) { 3042 /* 3043 * Hangul Composition. 3044 * 2. Two current code points are LV and T. 3045 */ 3046 int TIndex = uc2 - HC_TBASE; 3047 if (0 < TIndex && TIndex < HC_TCOUNT) { 3048 /* Make syllable of form LVT. */ 3049 UPDATE_UC(uc + TIndex); 3050 } else { 3051 WRITE_UC(); 3052 REPLACE_UC_WITH_UC2(); 3053 } 3054 continue; 3055 } else if ((nfc = get_nfc(uc, uc2)) != 0) { 3056 /* A composition to current code points 3057 * is found. */ 3058 UPDATE_UC(nfc); 3059 continue; 3060 } else if ((cl = CCC(uc2)) == 0) { 3061 /* Clearly 'uc2' the second code point is not 3062 * a decomposable code. */ 3063 WRITE_UC(); 3064 REPLACE_UC_WITH_UC2(); 3065 continue; 3066 } 3067 3068 /* 3069 * Collect following decomposable code points. 3070 */ 3071 cx = 0; 3072 ucx[0] = uc2; 3073 ccx[0] = cl; 3074 COLLECT_CPS(1); 3075 3076 /* 3077 * Find a composed code in the collected code points. 3078 */ 3079 i = 1; 3080 while (i < ucx_size) { 3081 int j; 3082 3083 if ((nfc = get_nfc(uc, ucx[i])) == 0) { 3084 i++; 3085 continue; 3086 } 3087 3088 /* 3089 * nfc is composed of uc and ucx[i]. 3090 */ 3091 UPDATE_UC(nfc); 3092 3093 /* 3094 * Remove ucx[i] by shifting 3095 * following code points. 3096 */ 3097 for (j = i; j+1 < ucx_size; j++) { 3098 ucx[j] = ucx[j+1]; 3099 ccx[j] = ccx[j+1]; 3100 } 3101 ucx_size --; 3102 3103 /* 3104 * Collect following code points blocked 3105 * by ucx[i] the removed code point. 3106 */ 3107 if (ucx_size > 0 && i == ucx_size && 3108 nx > 0 && cx == cl) { 3109 cl = ccx[ucx_size-1]; 3110 COLLECT_CPS(ucx_size); 3111 } 3112 /* 3113 * Restart finding a composed code with 3114 * the updated uc from the top of the 3115 * collected code points. 3116 */ 3117 i = 0; 3118 } 3119 3120 /* 3121 * Apparently the current code points are not 3122 * decomposed characters or already composed. 3123 */ 3124 WRITE_UC(); 3125 for (i = 0; i < ucx_size; i++) 3126 UNPARSE(p, endp, ucx[i]); 3127 3128 /* 3129 * Flush out remaining canonical combining characters. 3130 */ 3131 if (nx > 0 && cx == cl && len > 0) { 3132 while ((nx = parse(&ucx[0], s, len)) 3133 > 0) { 3134 cx = CCC(ucx[0]); 3135 if (cl > cx) 3136 break; 3137 s += nx; 3138 len -= nx; 3139 cl = cx; 3140 UNPARSE(p, endp, ucx[0]); 3141 } 3142 } 3143 break; 3144 } 3145 if (n2 < 0) { 3146 WRITE_UC(); 3147 /* Use a replaced unicode character. */ 3148 UNPARSE(p, endp, uc2); 3149 s += n2*-1; 3150 len -= n2*-1; 3151 ret = -1; 3152 continue; 3153 } else if (n2 == 0) { 3154 WRITE_UC(); 3155 break; 3156 } 3157 } 3158 as->length = p - as->s; 3159 as->s[as->length] = '\0'; 3160 if (ts == 2) 3161 as->s[as->length+1] = '\0'; 3162 return (ret); 3163 } 3164 3165 static int 3166 get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc) 3167 { 3168 int t, b; 3169 3170 /* 3171 * These are not converted to NFD on Mac OS. 3172 */ 3173 if ((uc >= 0x2000 && uc <= 0x2FFF) || 3174 (uc >= 0xF900 && uc <= 0xFAFF) || 3175 (uc >= 0x2F800 && uc <= 0x2FAFF)) 3176 return (0); 3177 /* 3178 * Those code points are not converted to NFD on Mac OS. 3179 * I do not know the reason because it is undocumented. 3180 * NFC NFD 3181 * 1109A ==> 11099 110BA 3182 * 1109C ==> 1109B 110BA 3183 * 110AB ==> 110A5 110BA 3184 */ 3185 if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB) 3186 return (0); 3187 3188 t = 0; 3189 b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1; 3190 while (b >= t) { 3191 int m = (t + b) / 2; 3192 if (u_decomposition_table[m].nfc < uc) 3193 t = m + 1; 3194 else if (u_decomposition_table[m].nfc > uc) 3195 b = m - 1; 3196 else { 3197 *cp1 = u_decomposition_table[m].cp1; 3198 *cp2 = u_decomposition_table[m].cp2; 3199 return (1); 3200 } 3201 } 3202 return (0); 3203 } 3204 3205 #define REPLACE_UC_WITH(cp) do { \ 3206 uc = cp; \ 3207 ucptr = NULL; \ 3208 } while (0) 3209 3210 /* 3211 * Normalize UTF-8 characters to Form D and copy the result. 3212 */ 3213 static int 3214 archive_string_normalize_D(struct archive_string *as, const void *_p, 3215 size_t len, struct archive_string_conv *sc) 3216 { 3217 const char *s = (const char *)_p; 3218 char *p, *endp; 3219 uint32_t uc, uc2; 3220 size_t w; 3221 int always_replace, n, n2, ret = 0, spair, ts, tm; 3222 int (*parse)(uint32_t *, const char *, size_t); 3223 size_t (*unparse)(char *, size_t, uint32_t); 3224 3225 always_replace = 1; 3226 ts = 1;/* text size. */ 3227 if (sc->flag & SCONV_TO_UTF16BE) { 3228 unparse = unicode_to_utf16be; 3229 ts = 2; 3230 if (sc->flag & SCONV_FROM_UTF16BE) 3231 always_replace = 0; 3232 } else if (sc->flag & SCONV_TO_UTF16LE) { 3233 unparse = unicode_to_utf16le; 3234 ts = 2; 3235 if (sc->flag & SCONV_FROM_UTF16LE) 3236 always_replace = 0; 3237 } else if (sc->flag & SCONV_TO_UTF8) { 3238 unparse = unicode_to_utf8; 3239 if (sc->flag & SCONV_FROM_UTF8) 3240 always_replace = 0; 3241 } else { 3242 /* 3243 * This case is going to be converted to another 3244 * character-set through iconv. 3245 */ 3246 always_replace = 0; 3247 if (sc->flag & SCONV_FROM_UTF16BE) { 3248 unparse = unicode_to_utf16be; 3249 ts = 2; 3250 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3251 unparse = unicode_to_utf16le; 3252 ts = 2; 3253 } else { 3254 unparse = unicode_to_utf8; 3255 } 3256 } 3257 3258 if (sc->flag & SCONV_FROM_UTF16BE) { 3259 parse = utf16be_to_unicode; 3260 tm = 1; 3261 spair = 4;/* surrogate pair size in UTF-16. */ 3262 } else if (sc->flag & SCONV_FROM_UTF16LE) { 3263 parse = utf16le_to_unicode; 3264 tm = 1; 3265 spair = 4;/* surrogate pair size in UTF-16. */ 3266 } else { 3267 parse = cesu8_to_unicode; 3268 tm = ts; 3269 spair = 6;/* surrogate pair size in UTF-8. */ 3270 } 3271 3272 if (archive_string_ensure(as, as->length + len * tm + ts) == NULL) 3273 return (-1); 3274 3275 p = as->s + as->length; 3276 endp = as->s + as->buffer_length - ts; 3277 while ((n = parse(&uc, s, len)) != 0) { 3278 const char *ucptr; 3279 uint32_t cp1, cp2; 3280 int SIndex; 3281 struct { 3282 uint32_t uc; 3283 int ccc; 3284 } fdc[FDC_MAX]; 3285 int fdi, fdj; 3286 int ccc; 3287 3288 check_first_code: 3289 if (n < 0) { 3290 /* Use a replaced unicode character. */ 3291 UNPARSE(p, endp, uc); 3292 s += n*-1; 3293 len -= n*-1; 3294 ret = -1; 3295 continue; 3296 } else if (n == spair || always_replace) 3297 /* uc is converted from a surrogate pair. 3298 * this should be treated as a changed code. */ 3299 ucptr = NULL; 3300 else 3301 ucptr = s; 3302 s += n; 3303 len -= n; 3304 3305 /* Hangul Decomposition. */ 3306 if ((SIndex = uc - HC_SBASE) >= 0 && SIndex < HC_SCOUNT) { 3307 int L = HC_LBASE + SIndex / HC_NCOUNT; 3308 int V = HC_VBASE + (SIndex % HC_NCOUNT) / HC_TCOUNT; 3309 int T = HC_TBASE + SIndex % HC_TCOUNT; 3310 3311 REPLACE_UC_WITH(L); 3312 WRITE_UC(); 3313 REPLACE_UC_WITH(V); 3314 WRITE_UC(); 3315 if (T != HC_TBASE) { 3316 REPLACE_UC_WITH(T); 3317 WRITE_UC(); 3318 } 3319 continue; 3320 } 3321 if (IS_DECOMPOSABLE_BLOCK(uc) && CCC(uc) != 0) { 3322 WRITE_UC(); 3323 continue; 3324 } 3325 3326 fdi = 0; 3327 while (get_nfd(&cp1, &cp2, uc) && fdi < FDC_MAX) { 3328 int k; 3329 3330 for (k = fdi; k > 0; k--) 3331 fdc[k] = fdc[k-1]; 3332 fdc[0].ccc = CCC(cp2); 3333 fdc[0].uc = cp2; 3334 fdi++; 3335 REPLACE_UC_WITH(cp1); 3336 } 3337 3338 /* Read following code points. */ 3339 while ((n2 = parse(&uc2, s, len)) > 0 && 3340 (ccc = CCC(uc2)) != 0 && fdi < FDC_MAX) { 3341 int j, k; 3342 3343 s += n2; 3344 len -= n2; 3345 for (j = 0; j < fdi; j++) { 3346 if (fdc[j].ccc > ccc) 3347 break; 3348 } 3349 if (j < fdi) { 3350 for (k = fdi; k > j; k--) 3351 fdc[k] = fdc[k-1]; 3352 fdc[j].ccc = ccc; 3353 fdc[j].uc = uc2; 3354 } else { 3355 fdc[fdi].ccc = ccc; 3356 fdc[fdi].uc = uc2; 3357 } 3358 fdi++; 3359 } 3360 3361 WRITE_UC(); 3362 for (fdj = 0; fdj < fdi; fdj++) { 3363 REPLACE_UC_WITH(fdc[fdj].uc); 3364 WRITE_UC(); 3365 } 3366 3367 if (n2 == 0) 3368 break; 3369 REPLACE_UC_WITH(uc2); 3370 n = n2; 3371 goto check_first_code; 3372 } 3373 as->length = p - as->s; 3374 as->s[as->length] = '\0'; 3375 if (ts == 2) 3376 as->s[as->length+1] = '\0'; 3377 return (ret); 3378 } 3379 3380 /* 3381 * libarchive 2.x made incorrect UTF-8 strings in the wrong assumption 3382 * that WCS is Unicode. It is true for several platforms but some are false. 3383 * And then people who did not use UTF-8 locale on the non Unicode WCS 3384 * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those 3385 * now cannot get right filename from libarchive 3.x and later since we 3386 * fixed the wrong assumption and it is incompatible to older its versions. 3387 * So we provide special option, "compat-2x.x", for resolving it. 3388 * That option enable the string conversion of libarchive 2.x. 3389 * 3390 * Translates the wrong UTF-8 string made by libarchive 2.x into current 3391 * locale character set and appends to the archive_string. 3392 * Note: returns -1 if conversion fails. 3393 */ 3394 static int 3395 strncat_from_utf8_libarchive2(struct archive_string *as, 3396 const void *_p, size_t len, struct archive_string_conv *sc) 3397 { 3398 const char *s; 3399 int n; 3400 char *p; 3401 char *end; 3402 uint32_t unicode; 3403 #if HAVE_WCRTOMB 3404 mbstate_t shift_state; 3405 3406 memset(&shift_state, 0, sizeof(shift_state)); 3407 #else 3408 /* Clear the shift state before starting. */ 3409 wctomb(NULL, L'\0'); 3410 #endif 3411 (void)sc; /* UNUSED */ 3412 /* 3413 * Allocate buffer for MBS. 3414 * We need this allocation here since it is possible that 3415 * as->s is still NULL. 3416 */ 3417 if (archive_string_ensure(as, as->length + len + 1) == NULL) 3418 return (-1); 3419 3420 s = (const char *)_p; 3421 p = as->s + as->length; 3422 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3423 while ((n = _utf8_to_unicode(&unicode, s, len)) != 0) { 3424 wchar_t wc; 3425 3426 if (p >= end) { 3427 as->length = p - as->s; 3428 /* Re-allocate buffer for MBS. */ 3429 if (archive_string_ensure(as, 3430 as->length + len * 2 + 1) == NULL) 3431 return (-1); 3432 p = as->s + as->length; 3433 end = as->s + as->buffer_length - MB_CUR_MAX -1; 3434 } 3435 3436 /* 3437 * As libarchie 2.x, translates the UTF-8 characters into 3438 * wide-characters in the assumption that WCS is Unicode. 3439 */ 3440 if (n < 0) { 3441 n *= -1; 3442 wc = L'?'; 3443 } else 3444 wc = (wchar_t)unicode; 3445 3446 s += n; 3447 len -= n; 3448 /* 3449 * Translates the wide-character into the current locale MBS. 3450 */ 3451 #if HAVE_WCRTOMB 3452 n = (int)wcrtomb(p, wc, &shift_state); 3453 #else 3454 n = (int)wctomb(p, wc); 3455 #endif 3456 if (n == -1) 3457 return (-1); 3458 p += n; 3459 } 3460 as->length = p - as->s; 3461 as->s[as->length] = '\0'; 3462 return (0); 3463 } 3464 3465 3466 /* 3467 * Conversion functions between current locale dependent MBS and UTF-16BE. 3468 * strncat_from_utf16be() : UTF-16BE --> MBS 3469 * strncat_to_utf16be() : MBS --> UTF16BE 3470 */ 3471 3472 #if defined(_WIN32) && !defined(__CYGWIN__) 3473 3474 /* 3475 * Convert a UTF-16BE/LE string to current locale and copy the result. 3476 * Return -1 if conversion failes. 3477 */ 3478 static int 3479 win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes, 3480 struct archive_string_conv *sc, int be) 3481 { 3482 struct archive_string tmp; 3483 const char *u16; 3484 int ll; 3485 BOOL defchar; 3486 char *mbs; 3487 size_t mbs_size, b; 3488 int ret = 0; 3489 3490 bytes &= ~1; 3491 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3492 return (-1); 3493 3494 mbs = as->s + as->length; 3495 mbs_size = as->buffer_length - as->length -1; 3496 3497 if (sc->to_cp == CP_C_LOCALE) { 3498 /* 3499 * "C" locale special process. 3500 */ 3501 u16 = _p; 3502 ll = 0; 3503 for (b = 0; b < bytes; b += 2) { 3504 uint16_t val; 3505 if (be) 3506 val = archive_be16dec(u16+b); 3507 else 3508 val = archive_le16dec(u16+b); 3509 if (val > 255) { 3510 *mbs++ = '?'; 3511 ret = -1; 3512 } else 3513 *mbs++ = (char)(val&0xff); 3514 ll++; 3515 } 3516 as->length += ll; 3517 as->s[as->length] = '\0'; 3518 return (ret); 3519 } 3520 3521 archive_string_init(&tmp); 3522 if (be) { 3523 if (is_big_endian()) { 3524 u16 = _p; 3525 } else { 3526 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3527 return (-1); 3528 memcpy(tmp.s, _p, bytes); 3529 for (b = 0; b < bytes; b += 2) { 3530 uint16_t val = archive_be16dec(tmp.s+b); 3531 archive_le16enc(tmp.s+b, val); 3532 } 3533 u16 = tmp.s; 3534 } 3535 } else { 3536 if (!is_big_endian()) { 3537 u16 = _p; 3538 } else { 3539 if (archive_string_ensure(&tmp, bytes+2) == NULL) 3540 return (-1); 3541 memcpy(tmp.s, _p, bytes); 3542 for (b = 0; b < bytes; b += 2) { 3543 uint16_t val = archive_le16dec(tmp.s+b); 3544 archive_be16enc(tmp.s+b, val); 3545 } 3546 u16 = tmp.s; 3547 } 3548 } 3549 3550 do { 3551 defchar = 0; 3552 ll = WideCharToMultiByte(sc->to_cp, 0, 3553 (LPCWSTR)u16, (int)bytes>>1, mbs, (int)mbs_size, 3554 NULL, &defchar); 3555 if (ll == 0 && 3556 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 3557 /* Need more buffer for MBS. */ 3558 ll = WideCharToMultiByte(sc->to_cp, 0, 3559 (LPCWSTR)u16, (int)bytes, NULL, 0, NULL, NULL); 3560 if (archive_string_ensure(as, ll +1) == NULL) 3561 return (-1); 3562 mbs = as->s + as->length; 3563 mbs_size = as->buffer_length - as->length -1; 3564 continue; 3565 } 3566 } while (0); 3567 archive_string_free(&tmp); 3568 as->length += ll; 3569 as->s[as->length] = '\0'; 3570 if (ll == 0 || defchar) 3571 ret = -1; 3572 return (ret); 3573 } 3574 3575 static int 3576 win_strncat_from_utf16be(struct archive_string *as, const void *_p, 3577 size_t bytes, struct archive_string_conv *sc) 3578 { 3579 return (win_strncat_from_utf16(as, _p, bytes, sc, 1)); 3580 } 3581 3582 static int 3583 win_strncat_from_utf16le(struct archive_string *as, const void *_p, 3584 size_t bytes, struct archive_string_conv *sc) 3585 { 3586 return (win_strncat_from_utf16(as, _p, bytes, sc, 0)); 3587 } 3588 3589 static int 3590 is_big_endian(void) 3591 { 3592 uint16_t d = 1; 3593 3594 return (archive_be16dec(&d) == 1); 3595 } 3596 3597 /* 3598 * Convert a current locale string to UTF-16BE/LE and copy the result. 3599 * Return -1 if conversion failes. 3600 */ 3601 static int 3602 win_strncat_to_utf16(struct archive_string *as16, const void *_p, 3603 size_t length, struct archive_string_conv *sc, int bigendian) 3604 { 3605 const char *s = (const char *)_p; 3606 char *u16; 3607 size_t count, avail; 3608 3609 if (archive_string_ensure(as16, 3610 as16->length + (length + 1) * 2) == NULL) 3611 return (-1); 3612 3613 u16 = as16->s + as16->length; 3614 avail = as16->buffer_length - 2; 3615 if (sc->from_cp == CP_C_LOCALE) { 3616 /* 3617 * "C" locale special process. 3618 */ 3619 count = 0; 3620 while (count < length && *s) { 3621 if (bigendian) 3622 archive_be16enc(u16, *s); 3623 else 3624 archive_le16enc(u16, *s); 3625 u16 += 2; 3626 s++; 3627 count++; 3628 } 3629 as16->length += count << 1; 3630 as16->s[as16->length] = 0; 3631 as16->s[as16->length+1] = 0; 3632 return (0); 3633 } 3634 do { 3635 count = MultiByteToWideChar(sc->from_cp, 3636 MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1); 3637 if (count == 0 && 3638 GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 3639 /* Need more buffer for UTF-16 string */ 3640 count = MultiByteToWideChar(sc->from_cp, 3641 MB_PRECOMPOSED, s, (int)length, NULL, 0); 3642 if (archive_string_ensure(as16, (count +1) * 2) 3643 == NULL) 3644 return (-1); 3645 u16 = as16->s + as16->length; 3646 avail = as16->buffer_length - 2; 3647 continue; 3648 } 3649 } while (0); 3650 as16->length += count * 2; 3651 as16->s[as16->length] = 0; 3652 as16->s[as16->length+1] = 0; 3653 if (count == 0) 3654 return (-1); 3655 3656 if (is_big_endian()) { 3657 if (!bigendian) { 3658 while (count > 0) { 3659 uint16_t v = archive_be16dec(u16); 3660 archive_le16enc(u16, v); 3661 u16 += 2; 3662 count--; 3663 } 3664 } 3665 } else { 3666 if (bigendian) { 3667 while (count > 0) { 3668 uint16_t v = archive_le16dec(u16); 3669 archive_be16enc(u16, v); 3670 u16 += 2; 3671 count--; 3672 } 3673 } 3674 } 3675 return (0); 3676 } 3677 3678 static int 3679 win_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3680 size_t length, struct archive_string_conv *sc) 3681 { 3682 return (win_strncat_to_utf16(as16, _p, length, sc, 1)); 3683 } 3684 3685 static int 3686 win_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3687 size_t length, struct archive_string_conv *sc) 3688 { 3689 return (win_strncat_to_utf16(as16, _p, length, sc, 0)); 3690 } 3691 3692 #endif /* _WIN32 && !__CYGWIN__ */ 3693 3694 /* 3695 * Do the best effort for conversions. 3696 * We cannot handle UTF-16BE character-set without such iconv, 3697 * but there is a chance if a string consists just ASCII code or 3698 * a current locale is UTF-8. 3699 */ 3700 3701 /* 3702 * Convert a UTF-16BE string to current locale and copy the result. 3703 * Return -1 if conversion failes. 3704 */ 3705 static int 3706 best_effort_strncat_from_utf16(struct archive_string *as, const void *_p, 3707 size_t bytes, struct archive_string_conv *sc, int be) 3708 { 3709 const char *utf16 = (const char *)_p; 3710 char *mbs; 3711 uint32_t uc; 3712 int n, ret; 3713 3714 (void)sc; /* UNUSED */ 3715 /* 3716 * Other case, we should do the best effort. 3717 * If all character are ASCII(<0x7f), we can convert it. 3718 * if not , we set a alternative character and return -1. 3719 */ 3720 ret = 0; 3721 if (archive_string_ensure(as, as->length + bytes +1) == NULL) 3722 return (-1); 3723 mbs = as->s + as->length; 3724 3725 while ((n = utf16_to_unicode(&uc, utf16, bytes, be)) != 0) { 3726 if (n < 0) { 3727 n *= -1; 3728 ret = -1; 3729 } 3730 bytes -= n; 3731 utf16 += n; 3732 3733 if (uc > 127) { 3734 /* We cannot handle it. */ 3735 *mbs++ = '?'; 3736 ret = -1; 3737 } else 3738 *mbs++ = (char)uc; 3739 } 3740 as->length = mbs - as->s; 3741 as->s[as->length] = '\0'; 3742 return (ret); 3743 } 3744 3745 static int 3746 best_effort_strncat_from_utf16be(struct archive_string *as, const void *_p, 3747 size_t bytes, struct archive_string_conv *sc) 3748 { 3749 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 1)); 3750 } 3751 3752 static int 3753 best_effort_strncat_from_utf16le(struct archive_string *as, const void *_p, 3754 size_t bytes, struct archive_string_conv *sc) 3755 { 3756 return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 0)); 3757 } 3758 3759 /* 3760 * Convert a current locale string to UTF-16BE/LE and copy the result. 3761 * Return -1 if conversion failes. 3762 */ 3763 static int 3764 best_effort_strncat_to_utf16(struct archive_string *as16, const void *_p, 3765 size_t length, struct archive_string_conv *sc, int bigendian) 3766 { 3767 const char *s = (const char *)_p; 3768 char *utf16; 3769 size_t remaining; 3770 int ret; 3771 3772 (void)sc; /* UNUSED */ 3773 /* 3774 * Other case, we should do the best effort. 3775 * If all character are ASCII(<0x7f), we can convert it. 3776 * if not , we set a alternative character and return -1. 3777 */ 3778 ret = 0; 3779 remaining = length; 3780 3781 if (archive_string_ensure(as16, 3782 as16->length + (length + 1) * 2) == NULL) 3783 return (-1); 3784 3785 utf16 = as16->s + as16->length; 3786 while (remaining--) { 3787 unsigned c = *s++; 3788 if (c > 127) { 3789 /* We cannot handle it. */ 3790 c = UNICODE_R_CHAR; 3791 ret = -1; 3792 } 3793 if (bigendian) 3794 archive_be16enc(utf16, c); 3795 else 3796 archive_le16enc(utf16, c); 3797 utf16 += 2; 3798 } 3799 as16->length = utf16 - as16->s; 3800 as16->s[as16->length] = 0; 3801 as16->s[as16->length+1] = 0; 3802 return (ret); 3803 } 3804 3805 static int 3806 best_effort_strncat_to_utf16be(struct archive_string *as16, const void *_p, 3807 size_t length, struct archive_string_conv *sc) 3808 { 3809 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 1)); 3810 } 3811 3812 static int 3813 best_effort_strncat_to_utf16le(struct archive_string *as16, const void *_p, 3814 size_t length, struct archive_string_conv *sc) 3815 { 3816 return (best_effort_strncat_to_utf16(as16, _p, length, sc, 0)); 3817 } 3818 3819 3820 /* 3821 * Multistring operations. 3822 */ 3823 3824 void 3825 archive_mstring_clean(struct archive_mstring *aes) 3826 { 3827 archive_wstring_free(&(aes->aes_wcs)); 3828 archive_string_free(&(aes->aes_mbs)); 3829 archive_string_free(&(aes->aes_utf8)); 3830 archive_string_free(&(aes->aes_mbs_in_locale)); 3831 aes->aes_set = 0; 3832 } 3833 3834 void 3835 archive_mstring_copy(struct archive_mstring *dest, struct archive_mstring *src) 3836 { 3837 dest->aes_set = src->aes_set; 3838 archive_string_copy(&(dest->aes_mbs), &(src->aes_mbs)); 3839 archive_string_copy(&(dest->aes_utf8), &(src->aes_utf8)); 3840 archive_wstring_copy(&(dest->aes_wcs), &(src->aes_wcs)); 3841 } 3842 3843 int 3844 archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes, 3845 const char **p) 3846 { 3847 struct archive_string_conv *sc; 3848 int r; 3849 3850 /* If we already have a UTF8 form, return that immediately. */ 3851 if (aes->aes_set & AES_SET_UTF8) { 3852 *p = aes->aes_utf8.s; 3853 return (0); 3854 } 3855 3856 *p = NULL; 3857 if (aes->aes_set & AES_SET_MBS) { 3858 sc = archive_string_conversion_to_charset(a, "UTF-8", 1); 3859 if (sc == NULL) 3860 return (-1);/* Couldn't allocate memory for sc. */ 3861 r = archive_strncpy_l(&(aes->aes_utf8), aes->aes_mbs.s, 3862 aes->aes_mbs.length, sc); 3863 if (a == NULL) 3864 free_sconv_object(sc); 3865 if (r == 0) { 3866 aes->aes_set |= AES_SET_UTF8; 3867 *p = aes->aes_utf8.s; 3868 return (0);/* success. */ 3869 } else 3870 return (-1);/* failure. */ 3871 } 3872 return (0);/* success. */ 3873 } 3874 3875 int 3876 archive_mstring_get_mbs(struct archive *a, struct archive_mstring *aes, 3877 const char **p) 3878 { 3879 int r, ret = 0; 3880 3881 (void)a; /* UNUSED */ 3882 /* If we already have an MBS form, return that immediately. */ 3883 if (aes->aes_set & AES_SET_MBS) { 3884 *p = aes->aes_mbs.s; 3885 return (ret); 3886 } 3887 3888 *p = NULL; 3889 /* If there's a WCS form, try converting with the native locale. */ 3890 if (aes->aes_set & AES_SET_WCS) { 3891 archive_string_empty(&(aes->aes_mbs)); 3892 r = archive_string_append_from_wcs(&(aes->aes_mbs), 3893 aes->aes_wcs.s, aes->aes_wcs.length); 3894 *p = aes->aes_mbs.s; 3895 if (r == 0) { 3896 aes->aes_set |= AES_SET_MBS; 3897 return (ret); 3898 } else 3899 ret = -1; 3900 } 3901 3902 /* 3903 * Only a UTF-8 form cannot avail because its conversion already 3904 * failed at archive_mstring_update_utf8(). 3905 */ 3906 return (ret); 3907 } 3908 3909 int 3910 archive_mstring_get_wcs(struct archive *a, struct archive_mstring *aes, 3911 const wchar_t **wp) 3912 { 3913 int r, ret = 0; 3914 3915 (void)a;/* UNUSED */ 3916 /* Return WCS form if we already have it. */ 3917 if (aes->aes_set & AES_SET_WCS) { 3918 *wp = aes->aes_wcs.s; 3919 return (ret); 3920 } 3921 3922 *wp = NULL; 3923 /* Try converting MBS to WCS using native locale. */ 3924 if (aes->aes_set & AES_SET_MBS) { 3925 archive_wstring_empty(&(aes->aes_wcs)); 3926 r = archive_wstring_append_from_mbs(&(aes->aes_wcs), 3927 aes->aes_mbs.s, aes->aes_mbs.length); 3928 if (r == 0) { 3929 aes->aes_set |= AES_SET_WCS; 3930 *wp = aes->aes_wcs.s; 3931 } else 3932 ret = -1;/* failure. */ 3933 } 3934 return (ret); 3935 } 3936 3937 int 3938 archive_mstring_get_mbs_l(struct archive_mstring *aes, 3939 const char **p, size_t *length, struct archive_string_conv *sc) 3940 { 3941 int r, ret = 0; 3942 3943 #if defined(_WIN32) && !defined(__CYGWIN__) 3944 /* 3945 * Internationalization programing on Windows must use Wide 3946 * characters because Windows platform cannot make locale UTF-8. 3947 */ 3948 if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) { 3949 archive_string_empty(&(aes->aes_mbs_in_locale)); 3950 r = archive_string_append_from_wcs_in_codepage( 3951 &(aes->aes_mbs_in_locale), aes->aes_wcs.s, 3952 aes->aes_wcs.length, sc); 3953 if (r == 0) { 3954 *p = aes->aes_mbs_in_locale.s; 3955 if (length != NULL) 3956 *length = aes->aes_mbs_in_locale.length; 3957 return (0); 3958 } else if (errno == ENOMEM) 3959 return (-1); 3960 else 3961 ret = -1; 3962 } 3963 #endif 3964 3965 /* If there is not an MBS form but is a WCS form, try converting 3966 * with the native locale to be used for translating it to specified 3967 * character-set. */ 3968 if ((aes->aes_set & AES_SET_MBS) == 0 && 3969 (aes->aes_set & AES_SET_WCS) != 0) { 3970 archive_string_empty(&(aes->aes_mbs)); 3971 r = archive_string_append_from_wcs(&(aes->aes_mbs), 3972 aes->aes_wcs.s, aes->aes_wcs.length); 3973 if (r == 0) 3974 aes->aes_set |= AES_SET_MBS; 3975 else if (errno == ENOMEM) 3976 return (-1); 3977 else 3978 ret = -1; 3979 } 3980 /* If we already have an MBS form, use it to be translated to 3981 * specified character-set. */ 3982 if (aes->aes_set & AES_SET_MBS) { 3983 if (sc == NULL) { 3984 /* Conversion is unneeded. */ 3985 *p = aes->aes_mbs.s; 3986 if (length != NULL) 3987 *length = aes->aes_mbs.length; 3988 return (0); 3989 } 3990 ret = archive_strncpy_l(&(aes->aes_mbs_in_locale), 3991 aes->aes_mbs.s, aes->aes_mbs.length, sc); 3992 *p = aes->aes_mbs_in_locale.s; 3993 if (length != NULL) 3994 *length = aes->aes_mbs_in_locale.length; 3995 } else { 3996 *p = NULL; 3997 if (length != NULL) 3998 *length = 0; 3999 } 4000 return (ret); 4001 } 4002 4003 int 4004 archive_mstring_copy_mbs(struct archive_mstring *aes, const char *mbs) 4005 { 4006 if (mbs == NULL) { 4007 aes->aes_set = 0; 4008 return (0); 4009 } 4010 return (archive_mstring_copy_mbs_len(aes, mbs, strlen(mbs))); 4011 } 4012 4013 int 4014 archive_mstring_copy_mbs_len(struct archive_mstring *aes, const char *mbs, 4015 size_t len) 4016 { 4017 if (mbs == NULL) { 4018 aes->aes_set = 0; 4019 return (0); 4020 } 4021 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4022 archive_strncpy(&(aes->aes_mbs), mbs, len); 4023 archive_string_empty(&(aes->aes_utf8)); 4024 archive_wstring_empty(&(aes->aes_wcs)); 4025 return (0); 4026 } 4027 4028 int 4029 archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs) 4030 { 4031 return archive_mstring_copy_wcs_len(aes, wcs, 4032 wcs == NULL ? 0 : wcslen(wcs)); 4033 } 4034 4035 int 4036 archive_mstring_copy_utf8(struct archive_mstring *aes, const char *utf8) 4037 { 4038 if (utf8 == NULL) { 4039 aes->aes_set = 0; 4040 } 4041 aes->aes_set = AES_SET_UTF8; 4042 archive_string_empty(&(aes->aes_mbs)); 4043 archive_string_empty(&(aes->aes_wcs)); 4044 archive_strncpy(&(aes->aes_utf8), utf8, strlen(utf8)); 4045 return (int)strlen(utf8); 4046 } 4047 4048 int 4049 archive_mstring_copy_wcs_len(struct archive_mstring *aes, const wchar_t *wcs, 4050 size_t len) 4051 { 4052 if (wcs == NULL) { 4053 aes->aes_set = 0; 4054 } 4055 aes->aes_set = AES_SET_WCS; /* Only WCS form set. */ 4056 archive_string_empty(&(aes->aes_mbs)); 4057 archive_string_empty(&(aes->aes_utf8)); 4058 archive_wstrncpy(&(aes->aes_wcs), wcs, len); 4059 return (0); 4060 } 4061 4062 int 4063 archive_mstring_copy_mbs_len_l(struct archive_mstring *aes, 4064 const char *mbs, size_t len, struct archive_string_conv *sc) 4065 { 4066 int r; 4067 4068 if (mbs == NULL) { 4069 aes->aes_set = 0; 4070 return (0); 4071 } 4072 archive_string_empty(&(aes->aes_mbs)); 4073 archive_wstring_empty(&(aes->aes_wcs)); 4074 archive_string_empty(&(aes->aes_utf8)); 4075 #if defined(_WIN32) && !defined(__CYGWIN__) 4076 /* 4077 * Internationalization programing on Windows must use Wide 4078 * characters because Windows platform cannot make locale UTF-8. 4079 */ 4080 if (sc == NULL) { 4081 if (archive_string_append(&(aes->aes_mbs), 4082 mbs, mbsnbytes(mbs, len)) == NULL) { 4083 aes->aes_set = 0; 4084 r = -1; 4085 } else { 4086 aes->aes_set = AES_SET_MBS; 4087 r = 0; 4088 } 4089 #if defined(HAVE_ICONV) 4090 } else if (sc != NULL && sc->cd_w != (iconv_t)-1) { 4091 /* 4092 * This case happens only when MultiByteToWideChar() cannot 4093 * handle sc->from_cp, and we have to iconv in order to 4094 * translate character-set to wchar_t,UTF-16. 4095 */ 4096 iconv_t cd = sc->cd; 4097 unsigned from_cp; 4098 int flag; 4099 4100 /* 4101 * Translate multi-bytes from some character-set to UTF-8. 4102 */ 4103 sc->cd = sc->cd_w; 4104 r = archive_strncpy_l(&(aes->aes_utf8), mbs, len, sc); 4105 sc->cd = cd; 4106 if (r != 0) { 4107 aes->aes_set = 0; 4108 return (r); 4109 } 4110 aes->aes_set = AES_SET_UTF8; 4111 4112 /* 4113 * Append the UTF-8 string into wstring. 4114 */ 4115 flag = sc->flag; 4116 sc->flag &= ~(SCONV_NORMALIZATION_C 4117 | SCONV_TO_UTF16| SCONV_FROM_UTF16); 4118 from_cp = sc->from_cp; 4119 sc->from_cp = CP_UTF8; 4120 r = archive_wstring_append_from_mbs_in_codepage(&(aes->aes_wcs), 4121 aes->aes_utf8.s, aes->aes_utf8.length, sc); 4122 sc->flag = flag; 4123 sc->from_cp = from_cp; 4124 if (r == 0) 4125 aes->aes_set |= AES_SET_WCS; 4126 #endif 4127 } else { 4128 r = archive_wstring_append_from_mbs_in_codepage( 4129 &(aes->aes_wcs), mbs, len, sc); 4130 if (r == 0) 4131 aes->aes_set = AES_SET_WCS; 4132 else 4133 aes->aes_set = 0; 4134 } 4135 #else 4136 r = archive_strncpy_l(&(aes->aes_mbs), mbs, len, sc); 4137 if (r == 0) 4138 aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */ 4139 else 4140 aes->aes_set = 0; 4141 #endif 4142 return (r); 4143 } 4144 4145 /* 4146 * The 'update' form tries to proactively update all forms of 4147 * this string (WCS and MBS) and returns an error if any of 4148 * them fail. This is used by the 'pax' handler, for instance, 4149 * to detect and report character-conversion failures early while 4150 * still allowing clients to get potentially useful values from 4151 * the more tolerant lazy conversions. (get_mbs and get_wcs will 4152 * strive to give the user something useful, so you can get hopefully 4153 * usable values even if some of the character conversions are failing.) 4154 */ 4155 int 4156 archive_mstring_update_utf8(struct archive *a, struct archive_mstring *aes, 4157 const char *utf8) 4158 { 4159 struct archive_string_conv *sc; 4160 int r; 4161 4162 if (utf8 == NULL) { 4163 aes->aes_set = 0; 4164 return (0); /* Succeeded in clearing everything. */ 4165 } 4166 4167 /* Save the UTF8 string. */ 4168 archive_strcpy(&(aes->aes_utf8), utf8); 4169 4170 /* Empty the mbs and wcs strings. */ 4171 archive_string_empty(&(aes->aes_mbs)); 4172 archive_wstring_empty(&(aes->aes_wcs)); 4173 4174 aes->aes_set = AES_SET_UTF8; /* Only UTF8 is set now. */ 4175 4176 /* Try converting UTF-8 to MBS, return false on failure. */ 4177 sc = archive_string_conversion_from_charset(a, "UTF-8", 1); 4178 if (sc == NULL) 4179 return (-1);/* Couldn't allocate memory for sc. */ 4180 r = archive_strcpy_l(&(aes->aes_mbs), utf8, sc); 4181 if (a == NULL) 4182 free_sconv_object(sc); 4183 if (r != 0) 4184 return (-1); 4185 aes->aes_set = AES_SET_UTF8 | AES_SET_MBS; /* Both UTF8 and MBS set. */ 4186 4187 /* Try converting MBS to WCS, return false on failure. */ 4188 if (archive_wstring_append_from_mbs(&(aes->aes_wcs), aes->aes_mbs.s, 4189 aes->aes_mbs.length)) 4190 return (-1); 4191 aes->aes_set = AES_SET_UTF8 | AES_SET_WCS | AES_SET_MBS; 4192 4193 /* All conversions succeeded. */ 4194 return (0); 4195 } 4196