1 /* Test of character set conversion with error handling and autodetection.
2 Copyright (C) 2007-2018 Free Software Foundation, Inc.
3
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 3 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <https://www.gnu.org/licenses/>. */
16
17 /* Written by Bruno Haible <bruno@clisp.org>, 2007. */
18
19 #include <config.h>
20
21 #include "striconveha.h"
22
23 #if HAVE_ICONV
24 # include <iconv.h>
25 #endif
26
27 #include <errno.h>
28 #include <stdlib.h>
29 #include <string.h>
30
31 #include "macros.h"
32 extern int iconv_supports_encoding (const char *encoding);
33
34 /* Magic number for detecting bounds violations. */
35 #define MAGIC 0x1983EFF1
36
37 static size_t *
new_offsets(size_t n)38 new_offsets (size_t n)
39 {
40 size_t *offsets = (size_t *) malloc ((n + 1) * sizeof (size_t));
41 offsets[n] = MAGIC;
42 return offsets;
43 }
44
45 int
main()46 main ()
47 {
48 static enum iconv_ilseq_handler handlers[] =
49 { iconveh_error, iconveh_question_mark, iconveh_escape_sequence };
50 size_t h;
51 size_t o;
52 size_t i;
53
54 #if HAVE_ICONV
55 /* Assume that iconv() supports at least the encodings ASCII, ISO-8859-1,
56 ISO-8859-2, and UTF-8. */
57
58 /* ------------------------- Test mem_iconveha() ------------------------- */
59
60 /* Test conversion from ISO-8859-2 to ISO-8859-1 with no errors. */
61 for (h = 0; h < SIZEOF (handlers); h++)
62 {
63 enum iconv_ilseq_handler handler = handlers[h];
64 static const char input[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
65 static const char expected[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
66 for (o = 0; o < 2; o++)
67 {
68 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
69 char *result = NULL;
70 size_t length = 0;
71 int retval = mem_iconveha (input, strlen (input),
72 "ISO-8859-2", "ISO-8859-1",
73 false, handler,
74 offsets,
75 &result, &length);
76 ASSERT (retval == 0);
77 ASSERT (length == strlen (expected));
78 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
79 if (o)
80 {
81 for (i = 0; i < 37; i++)
82 ASSERT (offsets[i] == i);
83 ASSERT (offsets[37] == MAGIC);
84 free (offsets);
85 }
86 free (result);
87 }
88 }
89
90 /* Test conversion from ISO-8859-2 to ISO-8859-1 with EILSEQ. */
91 for (h = 0; h < SIZEOF (handlers); h++)
92 {
93 enum iconv_ilseq_handler handler = handlers[h];
94 static const char input[] = "Rafa\263 Maszkowski"; /* Rafał Maszkowski */
95 for (o = 0; o < 2; o++)
96 {
97 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
98 char *result = NULL;
99 size_t length = 0;
100 int retval = mem_iconveha (input, strlen (input),
101 "ISO-8859-2", "ISO-8859-1",
102 false, handler,
103 offsets,
104 &result, &length);
105 switch (handler)
106 {
107 case iconveh_error:
108 ASSERT (retval == -1 && errno == EILSEQ);
109 ASSERT (result == NULL);
110 if (o)
111 free (offsets);
112 break;
113 case iconveh_question_mark:
114 {
115 static const char expected[] = "Rafa? Maszkowski";
116 ASSERT (retval == 0);
117 ASSERT (length == strlen (expected));
118 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
119 if (o)
120 {
121 for (i = 0; i < 16; i++)
122 ASSERT (offsets[i] == i);
123 ASSERT (offsets[16] == MAGIC);
124 free (offsets);
125 }
126 free (result);
127 }
128 break;
129 case iconveh_escape_sequence:
130 {
131 static const char expected[] = "Rafa\\u0142 Maszkowski";
132 ASSERT (retval == 0);
133 ASSERT (length == strlen (expected));
134 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
135 if (o)
136 {
137 for (i = 0; i < 16; i++)
138 ASSERT (offsets[i] == (i < 5 ? i :
139 i + 5));
140 ASSERT (offsets[16] == MAGIC);
141 free (offsets);
142 }
143 free (result);
144 }
145 break;
146 }
147 }
148 }
149
150 /* Test conversion from ISO-8859-1 to UTF-8 with no errors. */
151 for (h = 0; h < SIZEOF (handlers); h++)
152 {
153 enum iconv_ilseq_handler handler = handlers[h];
154 static const char input[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
155 static const char expected[] = "\303\204rger mit b\303\266sen B\303\274bchen ohne Augenma\303\237";
156 for (o = 0; o < 2; o++)
157 {
158 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
159 char *result = NULL;
160 size_t length = 0;
161 int retval = mem_iconveha (input, strlen (input),
162 "ISO-8859-1", "UTF-8",
163 false, handler,
164 offsets,
165 &result, &length);
166 ASSERT (retval == 0);
167 ASSERT (length == strlen (expected));
168 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
169 if (o)
170 {
171 for (i = 0; i < 37; i++)
172 ASSERT (offsets[i] == (i < 1 ? i :
173 i < 12 ? i + 1 :
174 i < 18 ? i + 2 :
175 i + 3));
176 ASSERT (offsets[37] == MAGIC);
177 free (offsets);
178 }
179 free (result);
180 }
181 }
182
183 /* Test conversion from UTF-8 to ISO-8859-1 with no errors. */
184 for (h = 0; h < SIZEOF (handlers); h++)
185 {
186 enum iconv_ilseq_handler handler = handlers[h];
187 static const char input[] = "\303\204rger mit b\303\266sen B\303\274bchen ohne Augenma\303\237";
188 static const char expected[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
189 for (o = 0; o < 2; o++)
190 {
191 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
192 char *result = NULL;
193 size_t length = 0;
194 int retval = mem_iconveha (input, strlen (input),
195 "UTF-8", "ISO-8859-1",
196 false, handler,
197 offsets,
198 &result, &length);
199 ASSERT (retval == 0);
200 ASSERT (length == strlen (expected));
201 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
202 if (o)
203 {
204 for (i = 0; i < 41; i++)
205 ASSERT (offsets[i] == (i < 1 ? i :
206 i == 1 ? (size_t)(-1) :
207 i < 13 ? i - 1 :
208 i == 13 ? (size_t)(-1) :
209 i < 20 ? i - 2 :
210 i == 20 ? (size_t)(-1) :
211 i < 40 ? i - 3 :
212 (size_t)(-1)));
213 ASSERT (offsets[41] == MAGIC);
214 free (offsets);
215 }
216 free (result);
217 }
218 }
219
220 /* Test conversion from UTF-8 to ISO-8859-1 with EILSEQ. */
221 for (h = 0; h < SIZEOF (handlers); h++)
222 {
223 enum iconv_ilseq_handler handler = handlers[h];
224 static const char input[] = "Rafa\305\202 Maszkowski"; /* Rafał Maszkowski */
225 for (o = 0; o < 2; o++)
226 {
227 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
228 char *result = NULL;
229 size_t length = 0;
230 int retval = mem_iconveha (input, strlen (input),
231 "UTF-8", "ISO-8859-1",
232 false, handler,
233 offsets,
234 &result, &length);
235 switch (handler)
236 {
237 case iconveh_error:
238 ASSERT (retval == -1 && errno == EILSEQ);
239 ASSERT (result == NULL);
240 if (o)
241 free (offsets);
242 break;
243 case iconveh_question_mark:
244 {
245 static const char expected[] = "Rafa? Maszkowski";
246 ASSERT (retval == 0);
247 ASSERT (length == strlen (expected));
248 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
249 if (o)
250 {
251 for (i = 0; i < 17; i++)
252 ASSERT (offsets[i] == (i < 5 ? i :
253 i == 5 ? (size_t)(-1) :
254 i - 1));
255 ASSERT (offsets[17] == MAGIC);
256 free (offsets);
257 }
258 free (result);
259 }
260 break;
261 case iconveh_escape_sequence:
262 {
263 static const char expected[] = "Rafa\\u0142 Maszkowski";
264 ASSERT (retval == 0);
265 ASSERT (length == strlen (expected));
266 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
267 if (o)
268 {
269 for (i = 0; i < 17; i++)
270 ASSERT (offsets[i] == (i < 5 ? i :
271 i == 5 ? (size_t)(-1) :
272 i + 4));
273 ASSERT (offsets[17] == MAGIC);
274 free (offsets);
275 }
276 free (result);
277 }
278 break;
279 }
280 }
281 }
282
283 /* Test conversion from UTF-8 to ISO-8859-1 with EINVAL. */
284 for (h = 0; h < SIZEOF (handlers); h++)
285 {
286 enum iconv_ilseq_handler handler = handlers[h];
287 static const char input[] = "\342";
288 for (o = 0; o < 2; o++)
289 {
290 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
291 char *result = NULL;
292 size_t length = 0;
293 int retval = mem_iconveha (input, strlen (input),
294 "UTF-8", "ISO-8859-1",
295 false, handler,
296 offsets,
297 &result, &length);
298 ASSERT (retval == 0);
299 ASSERT (length == 0);
300 if (o)
301 {
302 ASSERT (offsets[0] == 0);
303 ASSERT (offsets[1] == MAGIC);
304 free (offsets);
305 }
306 free (result);
307 }
308 }
309
310 /* autodetect_jp is only supported when iconv() support ISO-2022-JP-2. */
311 # if defined _LIBICONV_VERSION || !(defined _AIX || defined __sgi || defined __hpux || defined __osf__ || defined __sun)
312 if (iconv_supports_encoding ("ISO-2022-JP-2"))
313 {
314 /* Test conversions from autodetect_jp to UTF-8. */
315 for (h = 0; h < SIZEOF (handlers); h++)
316 {
317 enum iconv_ilseq_handler handler = handlers[h];
318 static const char input[] = "\244\263\244\363\244\313\244\301\244\317"; /* こんにちは in EUC-JP */
319 static const char expected[] = "\343\201\223\343\202\223\343\201\253\343\201\241\343\201\257"; /* こんにちは */
320 for (o = 0; o < 2; o++)
321 {
322 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
323 char *result = NULL;
324 size_t length = 0;
325 int retval = mem_iconveha (input, strlen (input),
326 "autodetect_jp", "UTF-8",
327 false, handler,
328 offsets,
329 &result, &length);
330 ASSERT (retval == 0);
331 ASSERT (length == strlen (expected));
332 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
333 if (o)
334 {
335 for (i = 0; i < 10; i++)
336 ASSERT (offsets[i] == ((i % 2) == 0 ? (i / 2) * 3 : (size_t)(-1)));
337 ASSERT (offsets[10] == MAGIC);
338 free (offsets);
339 }
340 free (result);
341 }
342 }
343 for (h = 0; h < SIZEOF (handlers); h++)
344 {
345 enum iconv_ilseq_handler handler = handlers[h];
346 static const char input[] = "\202\261\202\361\202\311\202\277\202\315"; /* こんにちは in Shift_JIS */
347 static const char expected[] = "\343\201\223\343\202\223\343\201\253\343\201\241\343\201\257"; /* こんにちは */
348 for (o = 0; o < 2; o++)
349 {
350 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
351 char *result = NULL;
352 size_t length = 0;
353 int retval = mem_iconveha (input, strlen (input),
354 "autodetect_jp", "UTF-8",
355 false, handler,
356 offsets,
357 &result, &length);
358 ASSERT (retval == 0);
359 ASSERT (length == strlen (expected));
360 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
361 if (o)
362 {
363 for (i = 0; i < 10; i++)
364 ASSERT (offsets[i] == ((i % 2) == 0 ? (i / 2) * 3 : (size_t)(-1)));
365 ASSERT (offsets[10] == MAGIC);
366 free (offsets);
367 }
368 free (result);
369 }
370 }
371 for (h = 0; h < SIZEOF (handlers); h++)
372 {
373 enum iconv_ilseq_handler handler = handlers[h];
374 static const char input[] = "\033$B$3$s$K$A$O\033(B"; /* こんにちは in ISO-2022-JP-2 */
375 static const char expected[] = "\343\201\223\343\202\223\343\201\253\343\201\241\343\201\257"; /* こんにちは */
376 for (o = 0; o < 2; o++)
377 {
378 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
379 char *result = NULL;
380 size_t length = 0;
381 int retval = mem_iconveha (input, strlen (input),
382 "autodetect_jp", "UTF-8",
383 false, handler,
384 offsets,
385 &result, &length);
386 ASSERT (retval == 0);
387 ASSERT (length == strlen (expected));
388 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
389 if (o)
390 {
391 for (i = 0; i < 16; i++)
392 ASSERT (offsets[i] == (i == 0 ? 0 :
393 i == 5 ? 3 :
394 i == 7 ? 6 :
395 i == 9 ? 9 :
396 i == 11 ? 12 :
397 i == 13 ? 15 :
398 (size_t)(-1)));
399 ASSERT (offsets[16] == MAGIC);
400 free (offsets);
401 }
402 free (result);
403 }
404 }
405 }
406 # endif
407
408 # if (((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2) || __GLIBC__ > 2) && !defined __UCLIBC__) || _LIBICONV_VERSION >= 0x0105
409 /* Test conversion from UTF-8 to ISO-8859-1 with transliteration. */
410 for (h = 0; h < SIZEOF (handlers); h++)
411 {
412 enum iconv_ilseq_handler handler = handlers[h];
413 static const char input[] = "Costs: 27 \342\202\254"; /* EURO SIGN */
414 static const char expected[] = "Costs: 27 EUR";
415 for (o = 0; o < 2; o++)
416 {
417 size_t *offsets = (o ? new_offsets (strlen (input)) : NULL);
418 char *result = NULL;
419 size_t length = 0;
420 int retval = mem_iconveha (input, strlen (input),
421 "UTF-8", "ISO-8859-1",
422 true, handler,
423 offsets,
424 &result, &length);
425 ASSERT (retval == 0);
426 ASSERT (length == strlen (expected));
427 ASSERT (result != NULL && memcmp (result, expected, strlen (expected)) == 0);
428 if (o)
429 {
430 for (i = 0; i < 13; i++)
431 ASSERT (offsets[i] == (i < 11 ? i : (size_t)(-1)));
432 ASSERT (offsets[13] == MAGIC);
433 free (offsets);
434 }
435 free (result);
436 }
437 }
438 # endif
439
440 /* ------------------------- Test str_iconveha() ------------------------- */
441
442 /* Test conversion from ISO-8859-2 to ISO-8859-1 with no errors. */
443 for (h = 0; h < SIZEOF (handlers); h++)
444 {
445 enum iconv_ilseq_handler handler = handlers[h];
446 static const char input[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
447 static const char expected[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
448 char *result = str_iconveha (input, "ISO-8859-2", "ISO-8859-1", false, handler);
449 ASSERT (result != NULL);
450 ASSERT (strcmp (result, expected) == 0);
451 free (result);
452 }
453
454 /* Test conversion from ISO-8859-2 to ISO-8859-1 with EILSEQ. */
455 for (h = 0; h < SIZEOF (handlers); h++)
456 {
457 enum iconv_ilseq_handler handler = handlers[h];
458 static const char input[] = "Rafa\263 Maszkowski"; /* Rafał Maszkowski */
459 char *result = str_iconveha (input, "ISO-8859-2", "ISO-8859-1", false, handler);
460 switch (handler)
461 {
462 case iconveh_error:
463 ASSERT (result == NULL && errno == EILSEQ);
464 break;
465 case iconveh_question_mark:
466 {
467 static const char expected[] = "Rafa? Maszkowski";
468 ASSERT (result != NULL);
469 ASSERT (strcmp (result, expected) == 0);
470 free (result);
471 }
472 break;
473 case iconveh_escape_sequence:
474 {
475 static const char expected[] = "Rafa\\u0142 Maszkowski";
476 ASSERT (result != NULL);
477 ASSERT (strcmp (result, expected) == 0);
478 free (result);
479 }
480 break;
481 }
482 }
483
484 /* Test conversion from ISO-8859-1 to UTF-8 with no errors. */
485 for (h = 0; h < SIZEOF (handlers); h++)
486 {
487 enum iconv_ilseq_handler handler = handlers[h];
488 static const char input[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
489 static const char expected[] = "\303\204rger mit b\303\266sen B\303\274bchen ohne Augenma\303\237";
490 char *result = str_iconveha (input, "ISO-8859-1", "UTF-8", false, handler);
491 ASSERT (result != NULL);
492 ASSERT (strcmp (result, expected) == 0);
493 free (result);
494 }
495
496 /* Test conversion from UTF-8 to ISO-8859-1 with no errors. */
497 for (h = 0; h < SIZEOF (handlers); h++)
498 {
499 enum iconv_ilseq_handler handler = handlers[h];
500 static const char input[] = "\303\204rger mit b\303\266sen B\303\274bchen ohne Augenma\303\237";
501 static const char expected[] = "\304rger mit b\366sen B\374bchen ohne Augenma\337";
502 char *result = str_iconveha (input, "UTF-8", "ISO-8859-1", false, handler);
503 ASSERT (result != NULL);
504 ASSERT (strcmp (result, expected) == 0);
505 free (result);
506 }
507
508 /* Test conversion from UTF-8 to ISO-8859-1 with EILSEQ. */
509 for (h = 0; h < SIZEOF (handlers); h++)
510 {
511 enum iconv_ilseq_handler handler = handlers[h];
512 static const char input[] = "Costs: 27 \342\202\254"; /* EURO SIGN */
513 char *result = str_iconveha (input, "UTF-8", "ISO-8859-1", false, handler);
514 switch (handler)
515 {
516 case iconveh_error:
517 ASSERT (result == NULL && errno == EILSEQ);
518 break;
519 case iconveh_question_mark:
520 {
521 static const char expected[] = "Costs: 27 ?";
522 ASSERT (result != NULL);
523 ASSERT (strcmp (result, expected) == 0);
524 free (result);
525 }
526 break;
527 case iconveh_escape_sequence:
528 {
529 static const char expected[] = "Costs: 27 \\u20AC";
530 ASSERT (result != NULL);
531 ASSERT (strcmp (result, expected) == 0);
532 free (result);
533 }
534 break;
535 }
536 }
537
538 /* Test conversion from UTF-8 to ISO-8859-1 with EINVAL. */
539 for (h = 0; h < SIZEOF (handlers); h++)
540 {
541 enum iconv_ilseq_handler handler = handlers[h];
542 static const char input[] = "\342";
543 char *result = str_iconveha (input, "UTF-8", "ISO-8859-1", false, handler);
544 ASSERT (result != NULL);
545 ASSERT (strcmp (result, "") == 0);
546 free (result);
547 }
548
549 /* autodetect_jp is only supported when iconv() support ISO-2022-JP-2. */
550 # if defined _LIBICONV_VERSION || !(defined _AIX || defined __sgi || defined __hpux || defined __osf__ || defined __sun)
551 if (iconv_supports_encoding ("ISO-2022-JP-2"))
552 {
553 /* Test conversions from autodetect_jp to UTF-8. */
554 for (h = 0; h < SIZEOF (handlers); h++)
555 {
556 enum iconv_ilseq_handler handler = handlers[h];
557 static const char input[] = "\244\263\244\363\244\313\244\301\244\317"; /* こんにちは in EUC-JP */
558 static const char expected[] = "\343\201\223\343\202\223\343\201\253\343\201\241\343\201\257"; /* こんにちは */
559 char *result = str_iconveha (input, "autodetect_jp", "UTF-8", false, handler);
560 ASSERT (result != NULL);
561 ASSERT (strcmp (result, expected) == 0);
562 free (result);
563 }
564 for (h = 0; h < SIZEOF (handlers); h++)
565 {
566 enum iconv_ilseq_handler handler = handlers[h];
567 static const char input[] = "\202\261\202\361\202\311\202\277\202\315"; /* こんにちは in Shift_JIS */
568 static const char expected[] = "\343\201\223\343\202\223\343\201\253\343\201\241\343\201\257"; /* こんにちは */
569 char *result = str_iconveha (input, "autodetect_jp", "UTF-8", false, handler);
570 ASSERT (result != NULL);
571 ASSERT (strcmp (result, expected) == 0);
572 free (result);
573 }
574 for (h = 0; h < SIZEOF (handlers); h++)
575 {
576 enum iconv_ilseq_handler handler = handlers[h];
577 static const char input[] = "\033$B$3$s$K$A$O\033(B"; /* こんにちは in ISO-2022-JP-2 */
578 static const char expected[] = "\343\201\223\343\202\223\343\201\253\343\201\241\343\201\257"; /* こんにちは */
579 char *result = str_iconveha (input, "autodetect_jp", "UTF-8", false, handler);
580 ASSERT (result != NULL);
581 ASSERT (strcmp (result, expected) == 0);
582 free (result);
583 }
584 }
585 # endif
586
587 # if (((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2) || __GLIBC__ > 2) && !defined __UCLIBC__) || _LIBICONV_VERSION >= 0x0105
588 /* Test conversion from UTF-8 to ISO-8859-1 with transliteration. */
589 for (h = 0; h < SIZEOF (handlers); h++)
590 {
591 enum iconv_ilseq_handler handler = handlers[h];
592 static const char input[] = "Costs: 27 \342\202\254"; /* EURO SIGN */
593 static const char expected[] = "Costs: 27 EUR";
594 char *result = str_iconveha (input, "UTF-8", "ISO-8859-1", true, handler);
595 ASSERT (result != NULL);
596 ASSERT (strcmp (result, expected) == 0);
597 free (result);
598 }
599 # endif
600
601 #endif
602
603 return 0;
604 }
605