1 /* $FreeBSD: head/lib/libiconv_modules/iconv_std/citrus_iconv_std.c 281550 2015-04-15 09:09:20Z tijl $ */
2 /*	$NetBSD: citrus_iconv_std.c,v 1.16 2012/02/12 13:51:29 wiz Exp $	*/
3 
4 /*-
5  * Copyright (c)2003 Citrus Project,
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 #include <sys/endian.h>
32 #include <sys/queue.h>
33 
34 #include <assert.h>
35 #include <errno.h>
36 #include <limits.h>
37 #include <stdbool.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 
42 #include "citrus_namespace.h"
43 #include "citrus_types.h"
44 #include "citrus_module.h"
45 #include "citrus_region.h"
46 #include "citrus_mmap.h"
47 #include "citrus_hash.h"
48 #include "citrus_iconv.h"
49 #include "citrus_stdenc.h"
50 #include "citrus_mapper.h"
51 #include "citrus_csmapper.h"
52 #include "citrus_memstream.h"
53 #include "citrus_iconv_std.h"
54 #include "citrus_esdb.h"
55 
56 /* ---------------------------------------------------------------------- */
57 
58 _CITRUS_ICONV_DECLS(iconv_std);
59 _CITRUS_ICONV_DEF_OPS(iconv_std);
60 
61 
62 /* ---------------------------------------------------------------------- */
63 
64 int
65 _citrus_iconv_std_iconv_getops(struct _citrus_iconv_ops *ops)
66 {
67 
68 	memcpy(ops, &_citrus_iconv_std_iconv_ops,
69 	    sizeof(_citrus_iconv_std_iconv_ops));
70 
71 	return (0);
72 }
73 
74 /* ---------------------------------------------------------------------- */
75 
76 /*
77  * convenience routines for stdenc.
78  */
79 static __inline void
80 save_encoding_state(struct _citrus_iconv_std_encoding *se)
81 {
82 
83 	if (se->se_ps)
84 		memcpy(se->se_pssaved, se->se_ps,
85 		    _stdenc_get_state_size(se->se_handle));
86 }
87 
88 static __inline void
89 restore_encoding_state(struct _citrus_iconv_std_encoding *se)
90 {
91 
92 	if (se->se_ps)
93 		memcpy(se->se_ps, se->se_pssaved,
94 		    _stdenc_get_state_size(se->se_handle));
95 }
96 
97 static __inline void
98 init_encoding_state(struct _citrus_iconv_std_encoding *se)
99 {
100 
101 	if (se->se_ps)
102 		_stdenc_init_state(se->se_handle, se->se_ps);
103 }
104 
105 static __inline int
106 mbtocsx(struct _citrus_iconv_std_encoding *se,
107     _csid_t *csid, _index_t *idx, char **s, size_t n, size_t *nresult,
108     struct iconv_hooks *hooks)
109 {
110 
111 	return (_stdenc_mbtocs(se->se_handle, csid, idx, s, n, se->se_ps,
112 			      nresult, hooks));
113 }
114 
115 static __inline int
116 cstombx(struct _citrus_iconv_std_encoding *se,
117     char *s, size_t n, _csid_t csid, _index_t idx, size_t *nresult,
118     struct iconv_hooks *hooks)
119 {
120 
121 	return (_stdenc_cstomb(se->se_handle, s, n, csid, idx, se->se_ps,
122 			      nresult, hooks));
123 }
124 
125 static __inline int
126 wctombx(struct _citrus_iconv_std_encoding *se,
127     char *s, size_t n, _wc_t wc, size_t *nresult,
128     struct iconv_hooks *hooks)
129 {
130 
131 	return (_stdenc_wctomb(se->se_handle, s, n, wc, se->se_ps, nresult,
132 			     hooks));
133 }
134 
135 static __inline int
136 put_state_resetx(struct _citrus_iconv_std_encoding *se, char *s, size_t n,
137     size_t *nresult)
138 {
139 
140 	return (_stdenc_put_state_reset(se->se_handle, s, n, se->se_ps, nresult));
141 }
142 
143 static __inline int
144 get_state_desc_gen(struct _citrus_iconv_std_encoding *se, int *rstate)
145 {
146 	struct _stdenc_state_desc ssd;
147 	int ret;
148 
149 	ret = _stdenc_get_state_desc(se->se_handle, se->se_ps,
150 	    _STDENC_SDID_GENERIC, &ssd);
151 	if (!ret)
152 		*rstate = ssd.u.generic.state;
153 
154 	return (ret);
155 }
156 
157 /*
158  * init encoding context
159  */
160 static int
161 init_encoding(struct _citrus_iconv_std_encoding *se, struct _stdenc *cs,
162     void *ps1, void *ps2)
163 {
164 	int ret = -1;
165 
166 	se->se_handle = cs;
167 	se->se_ps = ps1;
168 	se->se_pssaved = ps2;
169 
170 	if (se->se_ps)
171 		ret = _stdenc_init_state(cs, se->se_ps);
172 	if (!ret && se->se_pssaved)
173 		ret = _stdenc_init_state(cs, se->se_pssaved);
174 
175 	return (ret);
176 }
177 
178 static int
179 open_csmapper(struct _csmapper **rcm, const char *src, const char *dst,
180     unsigned long *rnorm)
181 {
182 	struct _csmapper *cm;
183 	int ret;
184 
185 	ret = _csmapper_open(&cm, src, dst, 0, rnorm);
186 	if (ret)
187 		return (ret);
188 	if (_csmapper_get_src_max(cm) != 1 || _csmapper_get_dst_max(cm) != 1 ||
189 	    _csmapper_get_state_size(cm) != 0) {
190 		_csmapper_close(cm);
191 		return (EINVAL);
192 	}
193 
194 	*rcm = cm;
195 
196 	return (0);
197 }
198 
199 static void
200 close_dsts(struct _citrus_iconv_std_dst_list *dl)
201 {
202 	struct _citrus_iconv_std_dst *sd;
203 
204 	while ((sd = TAILQ_FIRST(dl)) != NULL) {
205 		TAILQ_REMOVE(dl, sd, sd_entry);
206 		_csmapper_close(sd->sd_mapper);
207 		free(sd);
208 	}
209 }
210 
211 static int
212 open_dsts(struct _citrus_iconv_std_dst_list *dl,
213     const struct _esdb_charset *ec, const struct _esdb *dbdst)
214 {
215 	struct _citrus_iconv_std_dst *sd, *sdtmp;
216 	unsigned long norm;
217 	int i, ret;
218 
219 	sd = malloc(sizeof(*sd));
220 	if (sd == NULL)
221 		return (errno);
222 
223 	for (i = 0; i < dbdst->db_num_charsets; i++) {
224 		ret = open_csmapper(&sd->sd_mapper, ec->ec_csname,
225 		    dbdst->db_charsets[i].ec_csname, &norm);
226 		if (ret == 0) {
227 			sd->sd_csid = dbdst->db_charsets[i].ec_csid;
228 			sd->sd_norm = norm;
229 			/* insert this mapper by sorted order. */
230 			TAILQ_FOREACH(sdtmp, dl, sd_entry) {
231 				if (sdtmp->sd_norm > norm) {
232 					TAILQ_INSERT_BEFORE(sdtmp, sd,
233 					    sd_entry);
234 					sd = NULL;
235 					break;
236 				}
237 			}
238 			if (sd)
239 				TAILQ_INSERT_TAIL(dl, sd, sd_entry);
240 			sd = malloc(sizeof(*sd));
241 			if (sd == NULL) {
242 				ret = errno;
243 				close_dsts(dl);
244 				return (ret);
245 			}
246 		} else if (ret != ENOENT) {
247 			close_dsts(dl);
248 			free(sd);
249 			return (ret);
250 		}
251 	}
252 	free(sd);
253 	return (0);
254 }
255 
256 static void
257 close_srcs(struct _citrus_iconv_std_src_list *sl)
258 {
259 	struct _citrus_iconv_std_src *ss;
260 
261 	while ((ss = TAILQ_FIRST(sl)) != NULL) {
262 		TAILQ_REMOVE(sl, ss, ss_entry);
263 		close_dsts(&ss->ss_dsts);
264 		free(ss);
265 	}
266 }
267 
268 static int
269 open_srcs(struct _citrus_iconv_std_src_list *sl,
270     const struct _esdb *dbsrc, const struct _esdb *dbdst)
271 {
272 	struct _citrus_iconv_std_src *ss;
273 	int count = 0, i, ret;
274 
275 	ss = malloc(sizeof(*ss));
276 	if (ss == NULL)
277 		return (errno);
278 
279 	TAILQ_INIT(&ss->ss_dsts);
280 
281 	for (i = 0; i < dbsrc->db_num_charsets; i++) {
282 		ret = open_dsts(&ss->ss_dsts, &dbsrc->db_charsets[i], dbdst);
283 		if (ret)
284 			goto err;
285 		if (!TAILQ_EMPTY(&ss->ss_dsts)) {
286 			ss->ss_csid = dbsrc->db_charsets[i].ec_csid;
287 			TAILQ_INSERT_TAIL(sl, ss, ss_entry);
288 			ss = malloc(sizeof(*ss));
289 			if (ss == NULL) {
290 				ret = errno;
291 				goto err;
292 			}
293 			count++;
294 			TAILQ_INIT(&ss->ss_dsts);
295 		}
296 	}
297 	free(ss);
298 
299 	return (count ? 0 : ENOENT);
300 
301 err:
302 	free(ss);
303 	close_srcs(sl);
304 	return (ret);
305 }
306 
307 /* do convert a character */
308 #define E_NO_CORRESPONDING_CHAR ENOENT /* XXX */
309 static int
310 /*ARGSUSED*/
311 do_conv(const struct _citrus_iconv_std_shared *is,
312 	_csid_t *csid, _index_t *idx)
313 {
314 	struct _citrus_iconv_std_dst *sd;
315 	struct _citrus_iconv_std_src *ss;
316 	_index_t tmpidx;
317 	int ret;
318 
319 	TAILQ_FOREACH(ss, &is->is_srcs, ss_entry) {
320 		if (ss->ss_csid == *csid) {
321 			TAILQ_FOREACH(sd, &ss->ss_dsts, sd_entry) {
322 				ret = _csmapper_convert(sd->sd_mapper,
323 				    &tmpidx, *idx, NULL);
324 				switch (ret) {
325 				case _MAPPER_CONVERT_SUCCESS:
326 					*csid = sd->sd_csid;
327 					*idx = tmpidx;
328 					return (0);
329 				case _MAPPER_CONVERT_NONIDENTICAL:
330 					break;
331 				case _MAPPER_CONVERT_SRC_MORE:
332 					/*FALLTHROUGH*/
333 				case _MAPPER_CONVERT_DST_MORE:
334 					/*FALLTHROUGH*/
335 				case _MAPPER_CONVERT_ILSEQ:
336 					return (EILSEQ);
337 				case _MAPPER_CONVERT_FATAL:
338 					return (EINVAL);
339 				}
340 			}
341 			break;
342 		}
343 	}
344 
345 	return (E_NO_CORRESPONDING_CHAR);
346 }
347 /* ---------------------------------------------------------------------- */
348 
349 static int
350 /*ARGSUSED*/
351 _citrus_iconv_std_iconv_init_shared(struct _citrus_iconv_shared *ci,
352     const char * __restrict src, const char * __restrict dst)
353 {
354 	struct _citrus_esdb esdbdst, esdbsrc;
355 	struct _citrus_iconv_std_shared *is;
356 	int ret;
357 
358 	is = malloc(sizeof(*is));
359 	if (is == NULL) {
360 		ret = errno;
361 		goto err0;
362 	}
363 	ret = _citrus_esdb_open(&esdbsrc, src);
364 	if (ret)
365 		goto err1;
366 	ret = _citrus_esdb_open(&esdbdst, dst);
367 	if (ret)
368 		goto err2;
369 	ret = _stdenc_open(&is->is_src_encoding, esdbsrc.db_encname,
370 	    esdbsrc.db_variable, esdbsrc.db_len_variable);
371 	if (ret)
372 		goto err3;
373 	ret = _stdenc_open(&is->is_dst_encoding, esdbdst.db_encname,
374 	    esdbdst.db_variable, esdbdst.db_len_variable);
375 	if (ret)
376 		goto err4;
377 	is->is_use_invalid = esdbdst.db_use_invalid;
378 	is->is_invalid = esdbdst.db_invalid;
379 
380 	TAILQ_INIT(&is->is_srcs);
381 	ret = open_srcs(&is->is_srcs, &esdbsrc, &esdbdst);
382 	if (ret)
383 		goto err5;
384 
385 	_esdb_close(&esdbsrc);
386 	_esdb_close(&esdbdst);
387 	ci->ci_closure = is;
388 
389 	return (0);
390 
391 err5:
392 	_stdenc_close(is->is_dst_encoding);
393 err4:
394 	_stdenc_close(is->is_src_encoding);
395 err3:
396 	_esdb_close(&esdbdst);
397 err2:
398 	_esdb_close(&esdbsrc);
399 err1:
400 	free(is);
401 err0:
402 	return (ret);
403 }
404 
405 static void
406 _citrus_iconv_std_iconv_uninit_shared(struct _citrus_iconv_shared *ci)
407 {
408 	struct _citrus_iconv_std_shared *is = ci->ci_closure;
409 
410 	if (is == NULL)
411 		return;
412 
413 	_stdenc_close(is->is_src_encoding);
414 	_stdenc_close(is->is_dst_encoding);
415 	close_srcs(&is->is_srcs);
416 	free(is);
417 }
418 
419 static int
420 _citrus_iconv_std_iconv_init_context(struct _citrus_iconv *cv)
421 {
422 	const struct _citrus_iconv_std_shared *is = cv->cv_shared->ci_closure;
423 	struct _citrus_iconv_std_context *sc;
424 	char *ptr;
425 	size_t sz, szpsdst, szpssrc;
426 
427 	szpssrc = _stdenc_get_state_size(is->is_src_encoding);
428 	szpsdst = _stdenc_get_state_size(is->is_dst_encoding);
429 
430 	sz = (szpssrc + szpsdst)*2 + sizeof(struct _citrus_iconv_std_context);
431 	sc = malloc(sz);
432 	if (sc == NULL)
433 		return (errno);
434 
435 	ptr = (char *)&sc[1];
436 	if (szpssrc > 0)
437 		init_encoding(&sc->sc_src_encoding, is->is_src_encoding,
438 		    ptr, ptr+szpssrc);
439 	else
440 		init_encoding(&sc->sc_src_encoding, is->is_src_encoding,
441 		    NULL, NULL);
442 	ptr += szpssrc*2;
443 	if (szpsdst > 0)
444 		init_encoding(&sc->sc_dst_encoding, is->is_dst_encoding,
445 		    ptr, ptr+szpsdst);
446 	else
447 		init_encoding(&sc->sc_dst_encoding, is->is_dst_encoding,
448 		    NULL, NULL);
449 
450 	cv->cv_closure = (void *)sc;
451 
452 	return (0);
453 }
454 
455 static void
456 _citrus_iconv_std_iconv_uninit_context(struct _citrus_iconv *cv)
457 {
458 
459 	free(cv->cv_closure);
460 }
461 
462 static int
463 _citrus_iconv_std_iconv_convert(struct _citrus_iconv * __restrict cv,
464     char * __restrict * __restrict in, size_t * __restrict inbytes,
465     char * __restrict * __restrict out, size_t * __restrict outbytes,
466     uint32_t flags, size_t * __restrict invalids)
467 {
468 	const struct _citrus_iconv_std_shared *is = cv->cv_shared->ci_closure;
469 	struct _citrus_iconv_std_context *sc = cv->cv_closure;
470 	_csid_t csid;
471 	_index_t idx;
472 	char *tmpin;
473 	size_t inval, szrin, szrout;
474 	int ret, state = 0;
475 
476 	inval = 0;
477 	if (in == NULL || *in == NULL) {
478 		/* special cases */
479 		if (out != NULL && *out != NULL) {
480 			/* init output state and store the shift sequence */
481 			save_encoding_state(&sc->sc_src_encoding);
482 			save_encoding_state(&sc->sc_dst_encoding);
483 			szrout = 0;
484 
485 			ret = put_state_resetx(&sc->sc_dst_encoding,
486 			    *out, *outbytes, &szrout);
487 			if (ret)
488 				goto err;
489 
490 			if (szrout == (size_t)-2) {
491 				/* too small to store the character */
492 				ret = EINVAL;
493 				goto err;
494 			}
495 			*out += szrout;
496 			*outbytes -= szrout;
497 		} else
498 			/* otherwise, discard the shift sequence */
499 			init_encoding_state(&sc->sc_dst_encoding);
500 		init_encoding_state(&sc->sc_src_encoding);
501 		*invalids = 0;
502 		return (0);
503 	}
504 
505 	/* normal case */
506 	for (;;) {
507 		if (*inbytes == 0) {
508 			ret = get_state_desc_gen(&sc->sc_src_encoding, &state);
509 			if (state == _STDENC_SDGEN_INITIAL ||
510 			    state == _STDENC_SDGEN_STABLE)
511 				break;
512 		}
513 
514 		/* save the encoding states for the error recovery */
515 		save_encoding_state(&sc->sc_src_encoding);
516 		save_encoding_state(&sc->sc_dst_encoding);
517 
518 		/* mb -> csid/index */
519 		tmpin = *in;
520 		szrin = szrout = 0;
521 		ret = mbtocsx(&sc->sc_src_encoding, &csid, &idx, &tmpin,
522 		    *inbytes, &szrin, cv->cv_shared->ci_hooks);
523 		if (ret)
524 			goto err;
525 
526 		if (szrin == (size_t)-2) {
527 			/* incompleted character */
528 			ret = get_state_desc_gen(&sc->sc_src_encoding, &state);
529 			if (ret) {
530 				ret = EINVAL;
531 				goto err;
532 			}
533 			switch (state) {
534 			case _STDENC_SDGEN_INITIAL:
535 			case _STDENC_SDGEN_STABLE:
536 				/* fetch shift sequences only. */
537 				goto next;
538 			}
539 			ret = EINVAL;
540 			goto err;
541 		}
542 		/* convert the character */
543 		ret = do_conv(is, &csid, &idx);
544 		if (ret) {
545 			if (ret == E_NO_CORRESPONDING_CHAR) {
546 				/*
547 				 * GNU iconv returns EILSEQ when no
548 				 * corresponding character in the output.
549 				 * Some software depends on this behavior
550 				 * though this is against POSIX specification.
551 				 */
552 				if (cv->cv_shared->ci_ilseq_invalid != 0) {
553 					ret = EILSEQ;
554 					goto err;
555 				}
556 				inval++;
557 				szrout = 0;
558 				if ((((flags & _CITRUS_ICONV_F_HIDE_INVALID) == 0) &&
559 				    !cv->cv_shared->ci_discard_ilseq) &&
560 				    is->is_use_invalid) {
561 					ret = wctombx(&sc->sc_dst_encoding,
562 					    *out, *outbytes, is->is_invalid,
563 					    &szrout, cv->cv_shared->ci_hooks);
564 					if (ret)
565 						goto err;
566 				}
567 				goto next;
568 			} else
569 				goto err;
570 		}
571 		/* csid/index -> mb */
572 		ret = cstombx(&sc->sc_dst_encoding,
573 		    *out, *outbytes, csid, idx, &szrout,
574 		    cv->cv_shared->ci_hooks);
575 		if (ret)
576 			goto err;
577 next:
578 		*inbytes -= tmpin-*in; /* szrin is insufficient on \0. */
579 		*in = tmpin;
580 		*outbytes -= szrout;
581 		*out += szrout;
582 	}
583 	*invalids = inval;
584 
585 	return (0);
586 
587 err:
588 	restore_encoding_state(&sc->sc_src_encoding);
589 	restore_encoding_state(&sc->sc_dst_encoding);
590 	*invalids = inval;
591 
592 	return (ret);
593 }
594