1 /**********************************************************************
2
3 transcode.c -
4
5 $Author: usa $
6 created at: Tue Oct 30 16:10:22 JST 2007
7
8 Copyright (C) 2007 Martin Duerst
9
10 **********************************************************************/
11
12 #include "ruby/encoding.h"
13 #include "internal.h"
14 #include "transcode_data.h"
15 #include "id.h"
16 #include <ctype.h>
17
18 #define ENABLE_ECONV_NEWLINE_OPTION 1
19
20 /* VALUE rb_cEncoding = rb_define_class("Encoding", rb_cObject); */
21 static VALUE rb_eUndefinedConversionError;
22 static VALUE rb_eInvalidByteSequenceError;
23 static VALUE rb_eConverterNotFoundError;
24
25 VALUE rb_cEncodingConverter;
26
27 static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
28 static VALUE sym_xml, sym_text, sym_attr;
29 static VALUE sym_universal_newline;
30 static VALUE sym_crlf_newline;
31 static VALUE sym_cr_newline;
32 #ifdef ENABLE_ECONV_NEWLINE_OPTION
33 static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
34 #endif
35 static VALUE sym_partial_input;
36
37 static VALUE sym_invalid_byte_sequence;
38 static VALUE sym_undefined_conversion;
39 static VALUE sym_destination_buffer_full;
40 static VALUE sym_source_buffer_empty;
41 static VALUE sym_finished;
42 static VALUE sym_after_output;
43 static VALUE sym_incomplete_input;
44
45 static unsigned char *
46 allocate_converted_string(const char *sname, const char *dname,
47 const unsigned char *str, size_t len,
48 unsigned char *caller_dst_buf, size_t caller_dst_bufsize,
49 size_t *dst_len_ptr);
50
51 /* dynamic structure, one per conversion (similar to iconv_t) */
52 /* may carry conversion state (e.g. for iso-2022-jp) */
53 typedef struct rb_transcoding {
54 const rb_transcoder *transcoder;
55
56 int flags;
57
58 int resume_position;
59 unsigned int next_table;
60 VALUE next_info;
61 unsigned char next_byte;
62 unsigned int output_index;
63
64 ssize_t recognized_len; /* already interpreted */
65 ssize_t readagain_len; /* not yet interpreted */
66 union {
67 unsigned char ary[8]; /* max_input <= sizeof(ary) */
68 unsigned char *ptr; /* length: max_input */
69 } readbuf; /* recognized_len + readagain_len used */
70
71 ssize_t writebuf_off;
72 ssize_t writebuf_len;
73 union {
74 unsigned char ary[8]; /* max_output <= sizeof(ary) */
75 unsigned char *ptr; /* length: max_output */
76 } writebuf;
77
78 union rb_transcoding_state_t { /* opaque data for stateful encoding */
79 void *ptr;
80 char ary[sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*)];
81 double dummy_for_alignment;
82 } state;
83 } rb_transcoding;
84 #define TRANSCODING_READBUF(tc) \
85 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
86 (tc)->readbuf.ary : \
87 (tc)->readbuf.ptr)
88 #define TRANSCODING_WRITEBUF(tc) \
89 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
90 (tc)->writebuf.ary : \
91 (tc)->writebuf.ptr)
92 #define TRANSCODING_WRITEBUF_SIZE(tc) \
93 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
94 sizeof((tc)->writebuf.ary) : \
95 (size_t)(tc)->transcoder->max_output)
96 #define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
97 #define TRANSCODING_STATE(tc) \
98 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
99 (tc)->state.ary : \
100 (tc)->state.ptr)
101
102 typedef struct {
103 struct rb_transcoding *tc;
104 unsigned char *out_buf_start;
105 unsigned char *out_data_start;
106 unsigned char *out_data_end;
107 unsigned char *out_buf_end;
108 rb_econv_result_t last_result;
109 } rb_econv_elem_t;
110
111 struct rb_econv_t {
112 int flags;
113 int started; /* bool */
114
115 const char *source_encoding_name;
116 const char *destination_encoding_name;
117
118 const unsigned char *replacement_str;
119 size_t replacement_len;
120 const char *replacement_enc;
121
122 unsigned char *in_buf_start;
123 unsigned char *in_data_start;
124 unsigned char *in_data_end;
125 unsigned char *in_buf_end;
126 rb_econv_elem_t *elems;
127 int replacement_allocated; /* bool */
128 int num_allocated;
129 int num_trans;
130 int num_finished;
131 struct rb_transcoding *last_tc;
132
133 /* last error */
134 struct {
135 rb_econv_result_t result;
136 struct rb_transcoding *error_tc;
137 const char *source_encoding;
138 const char *destination_encoding;
139 const unsigned char *error_bytes_start;
140 size_t error_bytes_len;
141 size_t readagain_len;
142 } last_error;
143
144 /* The following fields are only for Encoding::Converter.
145 * rb_econv_open set them NULL. */
146 rb_encoding *source_encoding;
147 rb_encoding *destination_encoding;
148 };
149
150 /*
151 * Dispatch data and logic
152 */
153
154 #define DECORATOR_P(sname, dname) (*(sname) == '\0')
155
156 typedef struct {
157 const char *sname;
158 const char *dname;
159 const char *lib; /* null means no need to load a library */
160 const rb_transcoder *transcoder;
161 } transcoder_entry_t;
162
163 static st_table *transcoder_table;
164
165 static transcoder_entry_t *
make_transcoder_entry(const char * sname,const char * dname)166 make_transcoder_entry(const char *sname, const char *dname)
167 {
168 st_data_t val;
169 st_table *table2;
170
171 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
172 val = (st_data_t)st_init_strcasetable();
173 st_add_direct(transcoder_table, (st_data_t)sname, val);
174 }
175 table2 = (st_table *)val;
176 if (!st_lookup(table2, (st_data_t)dname, &val)) {
177 transcoder_entry_t *entry = ALLOC(transcoder_entry_t);
178 entry->sname = sname;
179 entry->dname = dname;
180 entry->lib = NULL;
181 entry->transcoder = NULL;
182 val = (st_data_t)entry;
183 st_add_direct(table2, (st_data_t)dname, val);
184 }
185 return (transcoder_entry_t *)val;
186 }
187
188 static transcoder_entry_t *
get_transcoder_entry(const char * sname,const char * dname)189 get_transcoder_entry(const char *sname, const char *dname)
190 {
191 st_data_t val;
192 st_table *table2;
193
194 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
195 return NULL;
196 }
197 table2 = (st_table *)val;
198 if (!st_lookup(table2, (st_data_t)dname, &val)) {
199 return NULL;
200 }
201 return (transcoder_entry_t *)val;
202 }
203
204 void
rb_register_transcoder(const rb_transcoder * tr)205 rb_register_transcoder(const rb_transcoder *tr)
206 {
207 const char *const sname = tr->src_encoding;
208 const char *const dname = tr->dst_encoding;
209
210 transcoder_entry_t *entry;
211
212 entry = make_transcoder_entry(sname, dname);
213 if (entry->transcoder) {
214 rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered",
215 sname, dname);
216 }
217
218 entry->transcoder = tr;
219 }
220
221 static void
declare_transcoder(const char * sname,const char * dname,const char * lib)222 declare_transcoder(const char *sname, const char *dname, const char *lib)
223 {
224 transcoder_entry_t *entry;
225
226 entry = make_transcoder_entry(sname, dname);
227 entry->lib = lib;
228 }
229
230 static const char transcoder_lib_prefix[] = "enc/trans/";
231
232 void
rb_declare_transcoder(const char * enc1,const char * enc2,const char * lib)233 rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
234 {
235 if (!lib) {
236 rb_raise(rb_eArgError, "invalid library name - (null)");
237 }
238 declare_transcoder(enc1, enc2, lib);
239 }
240
241 #define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
242
243 typedef struct search_path_queue_tag {
244 struct search_path_queue_tag *next;
245 const char *enc;
246 } search_path_queue_t;
247
248 typedef struct {
249 st_table *visited;
250 search_path_queue_t *queue;
251 search_path_queue_t **queue_last_ptr;
252 const char *base_enc;
253 } search_path_bfs_t;
254
255 static int
transcode_search_path_i(st_data_t key,st_data_t val,st_data_t arg)256 transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
257 {
258 const char *dname = (const char *)key;
259 search_path_bfs_t *bfs = (search_path_bfs_t *)arg;
260 search_path_queue_t *q;
261
262 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
263 return ST_CONTINUE;
264 }
265
266 q = ALLOC(search_path_queue_t);
267 q->enc = dname;
268 q->next = NULL;
269 *bfs->queue_last_ptr = q;
270 bfs->queue_last_ptr = &q->next;
271
272 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
273 return ST_CONTINUE;
274 }
275
276 static int
transcode_search_path(const char * sname,const char * dname,void (* callback)(const char * sname,const char * dname,int depth,void * arg),void * arg)277 transcode_search_path(const char *sname, const char *dname,
278 void (*callback)(const char *sname, const char *dname, int depth, void *arg),
279 void *arg)
280 {
281 search_path_bfs_t bfs;
282 search_path_queue_t *q;
283 st_data_t val;
284 st_table *table2;
285 int found;
286 int pathlen = -1;
287
288 if (encoding_equal(sname, dname))
289 return -1;
290
291 q = ALLOC(search_path_queue_t);
292 q->enc = sname;
293 q->next = NULL;
294 bfs.queue_last_ptr = &q->next;
295 bfs.queue = q;
296
297 bfs.visited = st_init_strcasetable();
298 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
299
300 while (bfs.queue) {
301 q = bfs.queue;
302 bfs.queue = q->next;
303 if (!bfs.queue)
304 bfs.queue_last_ptr = &bfs.queue;
305
306 if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) {
307 xfree(q);
308 continue;
309 }
310 table2 = (st_table *)val;
311
312 if (st_lookup(table2, (st_data_t)dname, &val)) {
313 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
314 xfree(q);
315 found = 1;
316 goto cleanup;
317 }
318
319 bfs.base_enc = q->enc;
320 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
321 bfs.base_enc = NULL;
322
323 xfree(q);
324 }
325 found = 0;
326
327 cleanup:
328 while (bfs.queue) {
329 q = bfs.queue;
330 bfs.queue = q->next;
331 xfree(q);
332 }
333
334 if (found) {
335 const char *enc = dname;
336 int depth;
337 pathlen = 0;
338 while (1) {
339 st_lookup(bfs.visited, (st_data_t)enc, &val);
340 if (!val)
341 break;
342 pathlen++;
343 enc = (const char *)val;
344 }
345 depth = pathlen;
346 enc = dname;
347 while (1) {
348 st_lookup(bfs.visited, (st_data_t)enc, &val);
349 if (!val)
350 break;
351 callback((const char *)val, enc, --depth, arg);
352 enc = (const char *)val;
353 }
354 }
355
356 st_free_table(bfs.visited);
357
358 return pathlen; /* is -1 if not found */
359 }
360
361 static const rb_transcoder *
load_transcoder_entry(transcoder_entry_t * entry)362 load_transcoder_entry(transcoder_entry_t *entry)
363 {
364 if (entry->transcoder)
365 return entry->transcoder;
366
367 if (entry->lib) {
368 const char *const lib = entry->lib;
369 const size_t len = strlen(lib);
370 const size_t total_len = sizeof(transcoder_lib_prefix) - 1 + len;
371 const VALUE fn = rb_str_new(0, total_len);
372 char *const path = RSTRING_PTR(fn);
373
374 memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1);
375 memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len);
376 rb_str_set_len(fn, total_len);
377 FL_UNSET(fn, FL_TAINT);
378 OBJ_FREEZE(fn);
379 rb_require_safe(fn, rb_safe_level());
380 }
381
382 if (entry->transcoder)
383 return entry->transcoder;
384
385 return NULL;
386 }
387
388 static const char*
get_replacement_character(const char * encname,size_t * len_ret,const char ** repl_encname_ptr)389 get_replacement_character(const char *encname, size_t *len_ret, const char **repl_encname_ptr)
390 {
391 if (encoding_equal(encname, "UTF-8")) {
392 *len_ret = 3;
393 *repl_encname_ptr = "UTF-8";
394 return "\xEF\xBF\xBD";
395 }
396 else {
397 *len_ret = 1;
398 *repl_encname_ptr = "US-ASCII";
399 return "?";
400 }
401 }
402
403 /*
404 * Transcoding engine logic
405 */
406
407 static const unsigned char *
transcode_char_start(rb_transcoding * tc,const unsigned char * in_start,const unsigned char * inchar_start,const unsigned char * in_p,size_t * char_len_ptr)408 transcode_char_start(rb_transcoding *tc,
409 const unsigned char *in_start,
410 const unsigned char *inchar_start,
411 const unsigned char *in_p,
412 size_t *char_len_ptr)
413 {
414 const unsigned char *ptr;
415 if (inchar_start - in_start < tc->recognized_len) {
416 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
417 inchar_start, unsigned char, in_p - inchar_start);
418 ptr = TRANSCODING_READBUF(tc);
419 }
420 else {
421 ptr = inchar_start - tc->recognized_len;
422 }
423 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
424 return ptr;
425 }
426
427 static rb_econv_result_t
transcode_restartable0(const unsigned char ** in_pos,unsigned char ** out_pos,const unsigned char * in_stop,unsigned char * out_stop,rb_transcoding * tc,const int opt)428 transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
429 const unsigned char *in_stop, unsigned char *out_stop,
430 rb_transcoding *tc,
431 const int opt)
432 {
433 const rb_transcoder *tr = tc->transcoder;
434 int unitlen = tr->input_unit_length;
435 ssize_t readagain_len = 0;
436
437 const unsigned char *inchar_start;
438 const unsigned char *in_p;
439
440 unsigned char *out_p;
441
442 in_p = inchar_start = *in_pos;
443
444 out_p = *out_pos;
445
446 #define SUSPEND(ret, num) \
447 do { \
448 tc->resume_position = (num); \
449 if (0 < in_p - inchar_start) \
450 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
451 inchar_start, unsigned char, in_p - inchar_start); \
452 *in_pos = in_p; \
453 *out_pos = out_p; \
454 tc->recognized_len += in_p - inchar_start; \
455 if (readagain_len) { \
456 tc->recognized_len -= readagain_len; \
457 tc->readagain_len = readagain_len; \
458 } \
459 return (ret); \
460 resume_label ## num:; \
461 } while (0)
462 #define SUSPEND_OBUF(num) \
463 do { \
464 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
465 } while (0)
466
467 #define SUSPEND_AFTER_OUTPUT(num) \
468 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
469 SUSPEND(econv_after_output, num); \
470 }
471
472 #define next_table (tc->next_table)
473 #define next_info (tc->next_info)
474 #define next_byte (tc->next_byte)
475 #define writebuf_len (tc->writebuf_len)
476 #define writebuf_off (tc->writebuf_off)
477
478 switch (tc->resume_position) {
479 case 0: break;
480 case 1: goto resume_label1;
481 case 2: goto resume_label2;
482 case 3: goto resume_label3;
483 case 4: goto resume_label4;
484 case 5: goto resume_label5;
485 case 6: goto resume_label6;
486 case 7: goto resume_label7;
487 case 8: goto resume_label8;
488 case 9: goto resume_label9;
489 case 10: goto resume_label10;
490 case 11: goto resume_label11;
491 case 12: goto resume_label12;
492 case 13: goto resume_label13;
493 case 14: goto resume_label14;
494 case 15: goto resume_label15;
495 case 16: goto resume_label16;
496 case 17: goto resume_label17;
497 case 18: goto resume_label18;
498 case 19: goto resume_label19;
499 case 20: goto resume_label20;
500 case 21: goto resume_label21;
501 case 22: goto resume_label22;
502 case 23: goto resume_label23;
503 case 24: goto resume_label24;
504 case 25: goto resume_label25;
505 case 26: goto resume_label26;
506 case 27: goto resume_label27;
507 case 28: goto resume_label28;
508 case 29: goto resume_label29;
509 case 30: goto resume_label30;
510 case 31: goto resume_label31;
511 case 32: goto resume_label32;
512 case 33: goto resume_label33;
513 case 34: goto resume_label34;
514 }
515
516 while (1) {
517 inchar_start = in_p;
518 tc->recognized_len = 0;
519 next_table = tr->conv_tree_start;
520
521 SUSPEND_AFTER_OUTPUT(24);
522
523 if (in_stop <= in_p) {
524 if (!(opt & ECONV_PARTIAL_INPUT))
525 break;
526 SUSPEND(econv_source_buffer_empty, 7);
527 continue;
528 }
529
530 #define BYTE_ADDR(index) (tr->byte_array + (index))
531 #define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
532 #define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
533 #define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
534 #define BL_MIN_BYTE (BL_BASE[0])
535 #define BL_MAX_BYTE (BL_BASE[1])
536 #define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
537 #define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
538
539 next_byte = (unsigned char)*in_p++;
540 follow_byte:
541 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
542 next_info = INVALID;
543 else {
544 next_info = (VALUE)BL_ACTION(next_byte);
545 }
546 follow_info:
547 switch (next_info & 0x1F) {
548 case NOMAP:
549 {
550 const unsigned char *p = inchar_start;
551 writebuf_off = 0;
552 while (p < in_p) {
553 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (unsigned char)*p++;
554 }
555 writebuf_len = writebuf_off;
556 writebuf_off = 0;
557 while (writebuf_off < writebuf_len) {
558 SUSPEND_OBUF(3);
559 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
560 }
561 }
562 continue;
563 case 0x00: case 0x04: case 0x08: case 0x0C:
564 case 0x10: case 0x14: case 0x18: case 0x1C:
565 SUSPEND_AFTER_OUTPUT(25);
566 while (in_p >= in_stop) {
567 if (!(opt & ECONV_PARTIAL_INPUT))
568 goto incomplete;
569 SUSPEND(econv_source_buffer_empty, 5);
570 }
571 next_byte = (unsigned char)*in_p++;
572 next_table = (unsigned int)next_info;
573 goto follow_byte;
574 case ZERObt: /* drop input */
575 continue;
576 case ONEbt:
577 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
578 continue;
579 case TWObt:
580 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
581 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
582 continue;
583 case THREEbt:
584 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
585 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
586 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
587 continue;
588 case FOURbt:
589 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
590 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
591 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
592 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
593 continue;
594 case GB4bt:
595 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
596 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
597 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
598 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
599 continue;
600 case STR1:
601 tc->output_index = 0;
602 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
603 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
604 tc->output_index++;
605 }
606 continue;
607 case FUNii:
608 next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info);
609 goto follow_info;
610 case FUNsi:
611 {
612 const unsigned char *char_start;
613 size_t char_len;
614 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
615 next_info = (VALUE)(*tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
616 goto follow_info;
617 }
618 case FUNio:
619 SUSPEND_OBUF(13);
620 if (tr->max_output <= out_stop - out_p)
621 out_p += tr->func_io(TRANSCODING_STATE(tc),
622 next_info, out_p, out_stop - out_p);
623 else {
624 writebuf_len = tr->func_io(TRANSCODING_STATE(tc),
625 next_info,
626 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
627 writebuf_off = 0;
628 while (writebuf_off < writebuf_len) {
629 SUSPEND_OBUF(20);
630 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
631 }
632 }
633 break;
634 case FUNso:
635 {
636 const unsigned char *char_start;
637 size_t char_len;
638 SUSPEND_OBUF(14);
639 if (tr->max_output <= out_stop - out_p) {
640 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
641 out_p += tr->func_so(TRANSCODING_STATE(tc),
642 char_start, (size_t)char_len,
643 out_p, out_stop - out_p);
644 }
645 else {
646 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
647 writebuf_len = tr->func_so(TRANSCODING_STATE(tc),
648 char_start, (size_t)char_len,
649 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
650 writebuf_off = 0;
651 while (writebuf_off < writebuf_len) {
652 SUSPEND_OBUF(22);
653 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
654 }
655 }
656 break;
657 }
658 case FUNsio:
659 {
660 const unsigned char *char_start;
661 size_t char_len;
662 SUSPEND_OBUF(33);
663 if (tr->max_output <= out_stop - out_p) {
664 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
665 out_p += tr->func_sio(TRANSCODING_STATE(tc),
666 char_start, (size_t)char_len, next_info,
667 out_p, out_stop - out_p);
668 }
669 else {
670 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
671 writebuf_len = tr->func_sio(TRANSCODING_STATE(tc),
672 char_start, (size_t)char_len, next_info,
673 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
674 writebuf_off = 0;
675 while (writebuf_off < writebuf_len) {
676 SUSPEND_OBUF(34);
677 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
678 }
679 }
680 break;
681 }
682 case INVALID:
683 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
684 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
685 SUSPEND_AFTER_OUTPUT(26);
686 while ((opt & ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
687 in_p = in_stop;
688 SUSPEND(econv_source_buffer_empty, 8);
689 }
690 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
691 in_p = in_stop;
692 }
693 else {
694 in_p = inchar_start + (unitlen - tc->recognized_len);
695 }
696 }
697 else {
698 ssize_t invalid_len; /* including the last byte which causes invalid */
699 ssize_t discard_len;
700 invalid_len = tc->recognized_len + (in_p - inchar_start);
701 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
702 readagain_len = invalid_len - discard_len;
703 }
704 goto invalid;
705 case UNDEF:
706 goto undef;
707 default:
708 rb_raise(rb_eRuntimeError, "unknown transcoding instruction");
709 }
710 continue;
711
712 invalid:
713 SUSPEND(econv_invalid_byte_sequence, 1);
714 continue;
715
716 incomplete:
717 SUSPEND(econv_incomplete_input, 27);
718 continue;
719
720 undef:
721 SUSPEND(econv_undefined_conversion, 2);
722 continue;
723 }
724
725 /* cleanup */
726 if (tr->finish_func) {
727 SUSPEND_OBUF(4);
728 if (tr->max_output <= out_stop - out_p) {
729 out_p += tr->finish_func(TRANSCODING_STATE(tc),
730 out_p, out_stop - out_p);
731 }
732 else {
733 writebuf_len = tr->finish_func(TRANSCODING_STATE(tc),
734 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
735 writebuf_off = 0;
736 while (writebuf_off < writebuf_len) {
737 SUSPEND_OBUF(23);
738 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
739 }
740 }
741 }
742 while (1)
743 SUSPEND(econv_finished, 6);
744 #undef SUSPEND
745 #undef next_table
746 #undef next_info
747 #undef next_byte
748 #undef writebuf_len
749 #undef writebuf_off
750 }
751
752 static rb_econv_result_t
transcode_restartable(const unsigned char ** in_pos,unsigned char ** out_pos,const unsigned char * in_stop,unsigned char * out_stop,rb_transcoding * tc,const int opt)753 transcode_restartable(const unsigned char **in_pos, unsigned char **out_pos,
754 const unsigned char *in_stop, unsigned char *out_stop,
755 rb_transcoding *tc,
756 const int opt)
757 {
758 if (tc->readagain_len) {
759 unsigned char *readagain_buf = ALLOCA_N(unsigned char, tc->readagain_len);
760 const unsigned char *readagain_pos = readagain_buf;
761 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
762 rb_econv_result_t res;
763
764 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
765 unsigned char, tc->readagain_len);
766 tc->readagain_len = 0;
767 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|ECONV_PARTIAL_INPUT);
768 if (res != econv_source_buffer_empty) {
769 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
770 readagain_pos, unsigned char, readagain_stop - readagain_pos);
771 tc->readagain_len += readagain_stop - readagain_pos;
772 return res;
773 }
774 }
775 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
776 }
777
778 static rb_transcoding *
rb_transcoding_open_by_transcoder(const rb_transcoder * tr,int flags)779 rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags)
780 {
781 rb_transcoding *tc;
782
783 tc = ALLOC(rb_transcoding);
784 tc->transcoder = tr;
785 tc->flags = flags;
786 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
787 tc->state.ptr = xmalloc(tr->state_size);
788 if (tr->state_init_func) {
789 (tr->state_init_func)(TRANSCODING_STATE(tc)); /* xxx: check return value */
790 }
791 tc->resume_position = 0;
792 tc->recognized_len = 0;
793 tc->readagain_len = 0;
794 tc->writebuf_len = 0;
795 tc->writebuf_off = 0;
796 if ((int)sizeof(tc->readbuf.ary) < tr->max_input) {
797 tc->readbuf.ptr = xmalloc(tr->max_input);
798 }
799 if ((int)sizeof(tc->writebuf.ary) < tr->max_output) {
800 tc->writebuf.ptr = xmalloc(tr->max_output);
801 }
802 return tc;
803 }
804
805 static rb_econv_result_t
rb_transcoding_convert(rb_transcoding * tc,const unsigned char ** input_ptr,const unsigned char * input_stop,unsigned char ** output_ptr,unsigned char * output_stop,int flags)806 rb_transcoding_convert(rb_transcoding *tc,
807 const unsigned char **input_ptr, const unsigned char *input_stop,
808 unsigned char **output_ptr, unsigned char *output_stop,
809 int flags)
810 {
811 return transcode_restartable(
812 input_ptr, output_ptr,
813 input_stop, output_stop,
814 tc, flags);
815 }
816
817 static void
rb_transcoding_close(rb_transcoding * tc)818 rb_transcoding_close(rb_transcoding *tc)
819 {
820 const rb_transcoder *tr = tc->transcoder;
821 if (tr->state_fini_func) {
822 (tr->state_fini_func)(TRANSCODING_STATE(tc)); /* check return value? */
823 }
824 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
825 xfree(tc->state.ptr);
826 if ((int)sizeof(tc->readbuf.ary) < tr->max_input)
827 xfree(tc->readbuf.ptr);
828 if ((int)sizeof(tc->writebuf.ary) < tr->max_output)
829 xfree(tc->writebuf.ptr);
830 xfree(tc);
831 }
832
833 static size_t
rb_transcoding_memsize(rb_transcoding * tc)834 rb_transcoding_memsize(rb_transcoding *tc)
835 {
836 size_t size = sizeof(rb_transcoding);
837 const rb_transcoder *tr = tc->transcoder;
838
839 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
840 size += tr->state_size;
841 }
842 if ((int)sizeof(tc->readbuf.ary) < tr->max_input) {
843 size += tr->max_input;
844 }
845 if ((int)sizeof(tc->writebuf.ary) < tr->max_output) {
846 size += tr->max_output;
847 }
848 return size;
849 }
850
851 static rb_econv_t *
rb_econv_alloc(int n_hint)852 rb_econv_alloc(int n_hint)
853 {
854 rb_econv_t *ec;
855
856 if (n_hint <= 0)
857 n_hint = 1;
858
859 ec = ALLOC(rb_econv_t);
860 ec->flags = 0;
861 ec->source_encoding_name = NULL;
862 ec->destination_encoding_name = NULL;
863 ec->started = 0;
864 ec->replacement_str = NULL;
865 ec->replacement_len = 0;
866 ec->replacement_enc = NULL;
867 ec->replacement_allocated = 0;
868 ec->in_buf_start = NULL;
869 ec->in_data_start = NULL;
870 ec->in_data_end = NULL;
871 ec->in_buf_end = NULL;
872 ec->num_allocated = n_hint;
873 ec->num_trans = 0;
874 ec->elems = ALLOC_N(rb_econv_elem_t, ec->num_allocated);
875 ec->num_finished = 0;
876 ec->last_tc = NULL;
877 ec->last_error.result = econv_source_buffer_empty;
878 ec->last_error.error_tc = NULL;
879 ec->last_error.source_encoding = NULL;
880 ec->last_error.destination_encoding = NULL;
881 ec->last_error.error_bytes_start = NULL;
882 ec->last_error.error_bytes_len = 0;
883 ec->last_error.readagain_len = 0;
884 ec->source_encoding = NULL;
885 ec->destination_encoding = NULL;
886 return ec;
887 }
888
889 static int
rb_econv_add_transcoder_at(rb_econv_t * ec,const rb_transcoder * tr,int i)890 rb_econv_add_transcoder_at(rb_econv_t *ec, const rb_transcoder *tr, int i)
891 {
892 int n, j;
893 int bufsize = 4096;
894 unsigned char *p;
895
896 if (ec->num_trans == ec->num_allocated) {
897 n = ec->num_allocated * 2;
898 REALLOC_N(ec->elems, rb_econv_elem_t, n);
899 ec->num_allocated = n;
900 }
901
902 p = xmalloc(bufsize);
903
904 MEMMOVE(ec->elems+i+1, ec->elems+i, rb_econv_elem_t, ec->num_trans-i);
905
906 ec->elems[i].tc = rb_transcoding_open_by_transcoder(tr, 0);
907 ec->elems[i].out_buf_start = p;
908 ec->elems[i].out_buf_end = p + bufsize;
909 ec->elems[i].out_data_start = p;
910 ec->elems[i].out_data_end = p;
911 ec->elems[i].last_result = econv_source_buffer_empty;
912
913 ec->num_trans++;
914
915 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding))
916 for (j = ec->num_trans-1; i <= j; j--) {
917 rb_transcoding *tc = ec->elems[j].tc;
918 const rb_transcoder *tr2 = tc->transcoder;
919 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
920 ec->last_tc = tc;
921 break;
922 }
923 }
924
925 return 0;
926 }
927
928 static rb_econv_t *
rb_econv_open_by_transcoder_entries(int n,transcoder_entry_t ** entries)929 rb_econv_open_by_transcoder_entries(int n, transcoder_entry_t **entries)
930 {
931 rb_econv_t *ec;
932 int i, ret;
933
934 for (i = 0; i < n; i++) {
935 const rb_transcoder *tr;
936 tr = load_transcoder_entry(entries[i]);
937 if (!tr)
938 return NULL;
939 }
940
941 ec = rb_econv_alloc(n);
942
943 for (i = 0; i < n; i++) {
944 const rb_transcoder *tr = load_transcoder_entry(entries[i]);
945 ret = rb_econv_add_transcoder_at(ec, tr, ec->num_trans);
946 if (ret == -1) {
947 rb_econv_close(ec);
948 return NULL;
949 }
950 }
951
952 return ec;
953 }
954
955 struct trans_open_t {
956 transcoder_entry_t **entries;
957 int num_additional;
958 };
959
960 static void
trans_open_i(const char * sname,const char * dname,int depth,void * arg)961 trans_open_i(const char *sname, const char *dname, int depth, void *arg)
962 {
963 struct trans_open_t *toarg = arg;
964
965 if (!toarg->entries) {
966 toarg->entries = ALLOC_N(transcoder_entry_t *, depth+1+toarg->num_additional);
967 }
968 toarg->entries[depth] = get_transcoder_entry(sname, dname);
969 }
970
971 static rb_econv_t *
rb_econv_open0(const char * sname,const char * dname,int ecflags)972 rb_econv_open0(const char *sname, const char *dname, int ecflags)
973 {
974 transcoder_entry_t **entries = NULL;
975 int num_trans;
976 rb_econv_t *ec;
977
978 int sidx, didx;
979
980 if (*sname) {
981 sidx = rb_enc_find_index(sname);
982 if (0 <= sidx) {
983 rb_enc_from_index(sidx);
984 }
985 }
986
987 if (*dname) {
988 didx = rb_enc_find_index(dname);
989 if (0 <= didx) {
990 rb_enc_from_index(didx);
991 }
992 }
993
994 if (*sname == '\0' && *dname == '\0') {
995 num_trans = 0;
996 entries = NULL;
997 sname = dname = "";
998 }
999 else {
1000 struct trans_open_t toarg;
1001 toarg.entries = NULL;
1002 toarg.num_additional = 0;
1003 num_trans = transcode_search_path(sname, dname, trans_open_i, (void *)&toarg);
1004 entries = toarg.entries;
1005 if (num_trans < 0) {
1006 xfree(entries);
1007 return NULL;
1008 }
1009 }
1010
1011 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1012 xfree(entries);
1013 if (!ec)
1014 return NULL;
1015
1016 ec->flags = ecflags;
1017 ec->source_encoding_name = sname;
1018 ec->destination_encoding_name = dname;
1019
1020 return ec;
1021 }
1022
1023 #define MAX_ECFLAGS_DECORATORS 32
1024
1025 static int
decorator_names(int ecflags,const char ** decorators_ret)1026 decorator_names(int ecflags, const char **decorators_ret)
1027 {
1028 int num_decorators;
1029
1030 switch (ecflags & ECONV_NEWLINE_DECORATOR_MASK) {
1031 case ECONV_UNIVERSAL_NEWLINE_DECORATOR:
1032 case ECONV_CRLF_NEWLINE_DECORATOR:
1033 case ECONV_CR_NEWLINE_DECORATOR:
1034 case 0:
1035 break;
1036 default:
1037 return -1;
1038 }
1039
1040 if ((ecflags & ECONV_XML_TEXT_DECORATOR) &&
1041 (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR))
1042 return -1;
1043
1044 num_decorators = 0;
1045
1046 if (ecflags & ECONV_XML_TEXT_DECORATOR)
1047 decorators_ret[num_decorators++] = "xml_text_escape";
1048 if (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR)
1049 decorators_ret[num_decorators++] = "xml_attr_content_escape";
1050 if (ecflags & ECONV_XML_ATTR_QUOTE_DECORATOR)
1051 decorators_ret[num_decorators++] = "xml_attr_quote";
1052
1053 if (ecflags & ECONV_CRLF_NEWLINE_DECORATOR)
1054 decorators_ret[num_decorators++] = "crlf_newline";
1055 if (ecflags & ECONV_CR_NEWLINE_DECORATOR)
1056 decorators_ret[num_decorators++] = "cr_newline";
1057 if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR)
1058 decorators_ret[num_decorators++] = "universal_newline";
1059
1060 return num_decorators;
1061 }
1062
1063 rb_econv_t *
rb_econv_open(const char * sname,const char * dname,int ecflags)1064 rb_econv_open(const char *sname, const char *dname, int ecflags)
1065 {
1066 rb_econv_t *ec;
1067 int num_decorators;
1068 const char *decorators[MAX_ECFLAGS_DECORATORS];
1069 int i;
1070
1071 num_decorators = decorator_names(ecflags, decorators);
1072 if (num_decorators == -1)
1073 return NULL;
1074
1075 ec = rb_econv_open0(sname, dname, ecflags & ECONV_ERROR_HANDLER_MASK);
1076 if (!ec)
1077 return NULL;
1078
1079 for (i = 0; i < num_decorators; i++)
1080 if (rb_econv_decorate_at_last(ec, decorators[i]) == -1) {
1081 rb_econv_close(ec);
1082 return NULL;
1083 }
1084
1085 ec->flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
1086
1087 return ec;
1088 }
1089
1090 static int
trans_sweep(rb_econv_t * ec,const unsigned char ** input_ptr,const unsigned char * input_stop,unsigned char ** output_ptr,unsigned char * output_stop,int flags,int start)1091 trans_sweep(rb_econv_t *ec,
1092 const unsigned char **input_ptr, const unsigned char *input_stop,
1093 unsigned char **output_ptr, unsigned char *output_stop,
1094 int flags,
1095 int start)
1096 {
1097 int try;
1098 int i, f;
1099
1100 const unsigned char **ipp, *is, *iold;
1101 unsigned char **opp, *os, *oold;
1102 rb_econv_result_t res;
1103
1104 try = 1;
1105 while (try) {
1106 try = 0;
1107 for (i = start; i < ec->num_trans; i++) {
1108 rb_econv_elem_t *te = &ec->elems[i];
1109
1110 if (i == 0) {
1111 ipp = input_ptr;
1112 is = input_stop;
1113 }
1114 else {
1115 rb_econv_elem_t *prev_te = &ec->elems[i-1];
1116 ipp = (const unsigned char **)&prev_te->out_data_start;
1117 is = prev_te->out_data_end;
1118 }
1119
1120 if (i == ec->num_trans-1) {
1121 opp = output_ptr;
1122 os = output_stop;
1123 }
1124 else {
1125 if (te->out_buf_start != te->out_data_start) {
1126 ssize_t len = te->out_data_end - te->out_data_start;
1127 ssize_t off = te->out_data_start - te->out_buf_start;
1128 MEMMOVE(te->out_buf_start, te->out_data_start, unsigned char, len);
1129 te->out_data_start = te->out_buf_start;
1130 te->out_data_end -= off;
1131 }
1132 opp = &te->out_data_end;
1133 os = te->out_buf_end;
1134 }
1135
1136 f = flags;
1137 if (ec->num_finished != i)
1138 f |= ECONV_PARTIAL_INPUT;
1139 if (i == 0 && (flags & ECONV_AFTER_OUTPUT)) {
1140 start = 1;
1141 flags &= ~ECONV_AFTER_OUTPUT;
1142 }
1143 if (i != 0)
1144 f &= ~ECONV_AFTER_OUTPUT;
1145 iold = *ipp;
1146 oold = *opp;
1147 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1148 if (iold != *ipp || oold != *opp)
1149 try = 1;
1150
1151 switch (res) {
1152 case econv_invalid_byte_sequence:
1153 case econv_incomplete_input:
1154 case econv_undefined_conversion:
1155 case econv_after_output:
1156 return i;
1157
1158 case econv_destination_buffer_full:
1159 case econv_source_buffer_empty:
1160 break;
1161
1162 case econv_finished:
1163 ec->num_finished = i+1;
1164 break;
1165 }
1166 }
1167 }
1168 return -1;
1169 }
1170
1171 static rb_econv_result_t
rb_trans_conv(rb_econv_t * ec,const unsigned char ** input_ptr,const unsigned char * input_stop,unsigned char ** output_ptr,unsigned char * output_stop,int flags,int * result_position_ptr)1172 rb_trans_conv(rb_econv_t *ec,
1173 const unsigned char **input_ptr, const unsigned char *input_stop,
1174 unsigned char **output_ptr, unsigned char *output_stop,
1175 int flags,
1176 int *result_position_ptr)
1177 {
1178 int i;
1179 int needreport_index;
1180 int sweep_start;
1181
1182 unsigned char empty_buf;
1183 unsigned char *empty_ptr = &empty_buf;
1184
1185 if (!input_ptr) {
1186 input_ptr = (const unsigned char **)&empty_ptr;
1187 input_stop = empty_ptr;
1188 }
1189
1190 if (!output_ptr) {
1191 output_ptr = &empty_ptr;
1192 output_stop = empty_ptr;
1193 }
1194
1195 if (ec->elems[0].last_result == econv_after_output)
1196 ec->elems[0].last_result = econv_source_buffer_empty;
1197
1198 needreport_index = -1;
1199 for (i = ec->num_trans-1; 0 <= i; i--) {
1200 switch (ec->elems[i].last_result) {
1201 case econv_invalid_byte_sequence:
1202 case econv_incomplete_input:
1203 case econv_undefined_conversion:
1204 case econv_after_output:
1205 case econv_finished:
1206 sweep_start = i+1;
1207 needreport_index = i;
1208 goto found_needreport;
1209
1210 case econv_destination_buffer_full:
1211 case econv_source_buffer_empty:
1212 break;
1213
1214 default:
1215 rb_bug("unexpected transcode last result");
1216 }
1217 }
1218
1219 /* /^[sd]+$/ is confirmed. but actually /^s*d*$/. */
1220
1221 if (ec->elems[ec->num_trans-1].last_result == econv_destination_buffer_full &&
1222 (flags & ECONV_AFTER_OUTPUT)) {
1223 rb_econv_result_t res;
1224
1225 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1226 (flags & ~ECONV_AFTER_OUTPUT)|ECONV_PARTIAL_INPUT,
1227 result_position_ptr);
1228
1229 if (res == econv_source_buffer_empty)
1230 return econv_after_output;
1231 return res;
1232 }
1233
1234 sweep_start = 0;
1235
1236 found_needreport:
1237
1238 do {
1239 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1240 sweep_start = needreport_index + 1;
1241 } while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1242
1243 for (i = ec->num_trans-1; 0 <= i; i--) {
1244 if (ec->elems[i].last_result != econv_source_buffer_empty) {
1245 rb_econv_result_t res = ec->elems[i].last_result;
1246 if (res == econv_invalid_byte_sequence ||
1247 res == econv_incomplete_input ||
1248 res == econv_undefined_conversion ||
1249 res == econv_after_output) {
1250 ec->elems[i].last_result = econv_source_buffer_empty;
1251 }
1252 if (result_position_ptr)
1253 *result_position_ptr = i;
1254 return res;
1255 }
1256 }
1257 if (result_position_ptr)
1258 *result_position_ptr = -1;
1259 return econv_source_buffer_empty;
1260 }
1261
1262 static rb_econv_result_t
rb_econv_convert0(rb_econv_t * ec,const unsigned char ** input_ptr,const unsigned char * input_stop,unsigned char ** output_ptr,unsigned char * output_stop,int flags)1263 rb_econv_convert0(rb_econv_t *ec,
1264 const unsigned char **input_ptr, const unsigned char *input_stop,
1265 unsigned char **output_ptr, unsigned char *output_stop,
1266 int flags)
1267 {
1268 rb_econv_result_t res;
1269 int result_position;
1270 int has_output = 0;
1271
1272 memset(&ec->last_error, 0, sizeof(ec->last_error));
1273
1274 if (ec->num_trans == 0) {
1275 size_t len;
1276 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1277 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1278 len = output_stop - *output_ptr;
1279 memcpy(*output_ptr, ec->in_data_start, len);
1280 *output_ptr = output_stop;
1281 ec->in_data_start += len;
1282 res = econv_destination_buffer_full;
1283 goto gotresult;
1284 }
1285 len = ec->in_data_end - ec->in_data_start;
1286 memcpy(*output_ptr, ec->in_data_start, len);
1287 *output_ptr += len;
1288 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1289 if (flags & ECONV_AFTER_OUTPUT) {
1290 res = econv_after_output;
1291 goto gotresult;
1292 }
1293 }
1294 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1295 len = output_stop - *output_ptr;
1296 }
1297 else {
1298 len = input_stop - *input_ptr;
1299 }
1300 if (0 < len && (flags & ECONV_AFTER_OUTPUT)) {
1301 *(*output_ptr)++ = *(*input_ptr)++;
1302 res = econv_after_output;
1303 goto gotresult;
1304 }
1305 memcpy(*output_ptr, *input_ptr, len);
1306 *output_ptr += len;
1307 *input_ptr += len;
1308 if (*input_ptr != input_stop)
1309 res = econv_destination_buffer_full;
1310 else if (flags & ECONV_PARTIAL_INPUT)
1311 res = econv_source_buffer_empty;
1312 else
1313 res = econv_finished;
1314 goto gotresult;
1315 }
1316
1317 if (ec->elems[ec->num_trans-1].out_data_start) {
1318 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1319 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1320 if (data_start != data_end) {
1321 size_t len;
1322 if (output_stop - *output_ptr < data_end - data_start) {
1323 len = output_stop - *output_ptr;
1324 memcpy(*output_ptr, data_start, len);
1325 *output_ptr = output_stop;
1326 ec->elems[ec->num_trans-1].out_data_start += len;
1327 res = econv_destination_buffer_full;
1328 goto gotresult;
1329 }
1330 len = data_end - data_start;
1331 memcpy(*output_ptr, data_start, len);
1332 *output_ptr += len;
1333 ec->elems[ec->num_trans-1].out_data_start =
1334 ec->elems[ec->num_trans-1].out_data_end =
1335 ec->elems[ec->num_trans-1].out_buf_start;
1336 has_output = 1;
1337 }
1338 }
1339
1340 if (ec->in_buf_start &&
1341 ec->in_data_start != ec->in_data_end) {
1342 res = rb_trans_conv(ec, (const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1343 (flags&~ECONV_AFTER_OUTPUT)|ECONV_PARTIAL_INPUT, &result_position);
1344 if (res != econv_source_buffer_empty)
1345 goto gotresult;
1346 }
1347
1348 if (has_output &&
1349 (flags & ECONV_AFTER_OUTPUT) &&
1350 *input_ptr != input_stop) {
1351 input_stop = *input_ptr;
1352 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1353 if (res == econv_source_buffer_empty)
1354 res = econv_after_output;
1355 }
1356 else if ((flags & ECONV_AFTER_OUTPUT) ||
1357 ec->num_trans == 1) {
1358 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1359 }
1360 else {
1361 flags |= ECONV_AFTER_OUTPUT;
1362 do {
1363 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1364 } while (res == econv_after_output);
1365 }
1366
1367 gotresult:
1368 ec->last_error.result = res;
1369 if (res == econv_invalid_byte_sequence ||
1370 res == econv_incomplete_input ||
1371 res == econv_undefined_conversion) {
1372 rb_transcoding *error_tc = ec->elems[result_position].tc;
1373 ec->last_error.error_tc = error_tc;
1374 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1375 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1376 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1377 ec->last_error.error_bytes_len = error_tc->recognized_len;
1378 ec->last_error.readagain_len = error_tc->readagain_len;
1379 }
1380
1381 return res;
1382 }
1383
1384 static int output_replacement_character(rb_econv_t *ec);
1385
1386 static int
output_hex_charref(rb_econv_t * ec)1387 output_hex_charref(rb_econv_t *ec)
1388 {
1389 int ret;
1390 unsigned char utfbuf[1024];
1391 const unsigned char *utf;
1392 size_t utf_len;
1393 int utf_allocated = 0;
1394 char charef_buf[16];
1395 const unsigned char *p;
1396
1397 if (encoding_equal(ec->last_error.source_encoding, "UTF-32BE")) {
1398 utf = ec->last_error.error_bytes_start;
1399 utf_len = ec->last_error.error_bytes_len;
1400 }
1401 else {
1402 utf = allocate_converted_string(ec->last_error.source_encoding, "UTF-32BE",
1403 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1404 utfbuf, sizeof(utfbuf),
1405 &utf_len);
1406 if (!utf)
1407 return -1;
1408 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1409 utf_allocated = 1;
1410 }
1411
1412 if (utf_len % 4 != 0)
1413 goto fail;
1414
1415 p = utf;
1416 while (4 <= utf_len) {
1417 unsigned int u = 0;
1418 u += p[0] << 24;
1419 u += p[1] << 16;
1420 u += p[2] << 8;
1421 u += p[3];
1422 snprintf(charef_buf, sizeof(charef_buf), "&#x%X;", u);
1423
1424 ret = rb_econv_insert_output(ec, (unsigned char *)charef_buf, strlen(charef_buf), "US-ASCII");
1425 if (ret == -1)
1426 goto fail;
1427
1428 p += 4;
1429 utf_len -= 4;
1430 }
1431
1432 if (utf_allocated)
1433 xfree((void *)utf);
1434 return 0;
1435
1436 fail:
1437 if (utf_allocated)
1438 xfree((void *)utf);
1439 return -1;
1440 }
1441
1442 rb_econv_result_t
rb_econv_convert(rb_econv_t * ec,const unsigned char ** input_ptr,const unsigned char * input_stop,unsigned char ** output_ptr,unsigned char * output_stop,int flags)1443 rb_econv_convert(rb_econv_t *ec,
1444 const unsigned char **input_ptr, const unsigned char *input_stop,
1445 unsigned char **output_ptr, unsigned char *output_stop,
1446 int flags)
1447 {
1448 rb_econv_result_t ret;
1449
1450 unsigned char empty_buf;
1451 unsigned char *empty_ptr = &empty_buf;
1452
1453 ec->started = 1;
1454
1455 if (!input_ptr) {
1456 input_ptr = (const unsigned char **)&empty_ptr;
1457 input_stop = empty_ptr;
1458 }
1459
1460 if (!output_ptr) {
1461 output_ptr = &empty_ptr;
1462 output_stop = empty_ptr;
1463 }
1464
1465 resume:
1466 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1467
1468 if (ret == econv_invalid_byte_sequence ||
1469 ret == econv_incomplete_input) {
1470 /* deal with invalid byte sequence */
1471 /* todo: add more alternative behaviors */
1472 switch (ec->flags & ECONV_INVALID_MASK) {
1473 case ECONV_INVALID_REPLACE:
1474 if (output_replacement_character(ec) == 0)
1475 goto resume;
1476 }
1477 }
1478
1479 if (ret == econv_undefined_conversion) {
1480 /* valid character in source encoding
1481 * but no related character(s) in destination encoding */
1482 /* todo: add more alternative behaviors */
1483 switch (ec->flags & ECONV_UNDEF_MASK) {
1484 case ECONV_UNDEF_REPLACE:
1485 if (output_replacement_character(ec) == 0)
1486 goto resume;
1487 break;
1488
1489 case ECONV_UNDEF_HEX_CHARREF:
1490 if (output_hex_charref(ec) == 0)
1491 goto resume;
1492 break;
1493 }
1494 }
1495
1496 return ret;
1497 }
1498
1499 const char *
rb_econv_encoding_to_insert_output(rb_econv_t * ec)1500 rb_econv_encoding_to_insert_output(rb_econv_t *ec)
1501 {
1502 rb_transcoding *tc = ec->last_tc;
1503 const rb_transcoder *tr;
1504
1505 if (tc == NULL)
1506 return "";
1507
1508 tr = tc->transcoder;
1509
1510 if (tr->asciicompat_type == asciicompat_encoder)
1511 return tr->src_encoding;
1512 return tr->dst_encoding;
1513 }
1514
1515 static unsigned char *
allocate_converted_string(const char * sname,const char * dname,const unsigned char * str,size_t len,unsigned char * caller_dst_buf,size_t caller_dst_bufsize,size_t * dst_len_ptr)1516 allocate_converted_string(const char *sname, const char *dname,
1517 const unsigned char *str, size_t len,
1518 unsigned char *caller_dst_buf, size_t caller_dst_bufsize,
1519 size_t *dst_len_ptr)
1520 {
1521 unsigned char *dst_str;
1522 size_t dst_len;
1523 size_t dst_bufsize;
1524
1525 rb_econv_t *ec;
1526 rb_econv_result_t res;
1527
1528 const unsigned char *sp;
1529 unsigned char *dp;
1530
1531 if (caller_dst_buf)
1532 dst_bufsize = caller_dst_bufsize;
1533 else if (len == 0)
1534 dst_bufsize = 1;
1535 else
1536 dst_bufsize = len;
1537
1538 ec = rb_econv_open(sname, dname, 0);
1539 if (ec == NULL)
1540 return NULL;
1541 if (caller_dst_buf)
1542 dst_str = caller_dst_buf;
1543 else
1544 dst_str = xmalloc(dst_bufsize);
1545 dst_len = 0;
1546 sp = str;
1547 dp = dst_str+dst_len;
1548 res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0);
1549 dst_len = dp - dst_str;
1550 while (res == econv_destination_buffer_full) {
1551 if (SIZE_MAX/2 < dst_bufsize) {
1552 goto fail;
1553 }
1554 dst_bufsize *= 2;
1555 if (dst_str == caller_dst_buf) {
1556 unsigned char *tmp;
1557 tmp = xmalloc(dst_bufsize);
1558 memcpy(tmp, dst_str, dst_bufsize/2);
1559 dst_str = tmp;
1560 }
1561 else {
1562 dst_str = xrealloc(dst_str, dst_bufsize);
1563 }
1564 dp = dst_str+dst_len;
1565 res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0);
1566 dst_len = dp - dst_str;
1567 }
1568 if (res != econv_finished) {
1569 goto fail;
1570 }
1571 rb_econv_close(ec);
1572 *dst_len_ptr = dst_len;
1573 return dst_str;
1574
1575 fail:
1576 if (dst_str != caller_dst_buf)
1577 xfree(dst_str);
1578 rb_econv_close(ec);
1579 return NULL;
1580 }
1581
1582 /* result: 0:success -1:failure */
1583 int
rb_econv_insert_output(rb_econv_t * ec,const unsigned char * str,size_t len,const char * str_encoding)1584 rb_econv_insert_output(rb_econv_t *ec,
1585 const unsigned char *str, size_t len, const char *str_encoding)
1586 {
1587 const char *insert_encoding = rb_econv_encoding_to_insert_output(ec);
1588 unsigned char insert_buf[4096];
1589 const unsigned char *insert_str = NULL;
1590 size_t insert_len;
1591
1592 int last_trans_index;
1593 rb_transcoding *tc;
1594
1595 unsigned char **buf_start_p;
1596 unsigned char **data_start_p;
1597 unsigned char **data_end_p;
1598 unsigned char **buf_end_p;
1599
1600 size_t need;
1601
1602 ec->started = 1;
1603
1604 if (len == 0)
1605 return 0;
1606
1607 if (encoding_equal(insert_encoding, str_encoding)) {
1608 insert_str = str;
1609 insert_len = len;
1610 }
1611 else {
1612 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1613 str, len, insert_buf, sizeof(insert_buf), &insert_len);
1614 if (insert_str == NULL)
1615 return -1;
1616 }
1617
1618 need = insert_len;
1619
1620 last_trans_index = ec->num_trans-1;
1621 if (ec->num_trans == 0) {
1622 tc = NULL;
1623 buf_start_p = &ec->in_buf_start;
1624 data_start_p = &ec->in_data_start;
1625 data_end_p = &ec->in_data_end;
1626 buf_end_p = &ec->in_buf_end;
1627 }
1628 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1629 tc = ec->elems[last_trans_index].tc;
1630 need += tc->readagain_len;
1631 if (need < insert_len)
1632 goto fail;
1633 if (last_trans_index == 0) {
1634 buf_start_p = &ec->in_buf_start;
1635 data_start_p = &ec->in_data_start;
1636 data_end_p = &ec->in_data_end;
1637 buf_end_p = &ec->in_buf_end;
1638 }
1639 else {
1640 rb_econv_elem_t *ee = &ec->elems[last_trans_index-1];
1641 buf_start_p = &ee->out_buf_start;
1642 data_start_p = &ee->out_data_start;
1643 data_end_p = &ee->out_data_end;
1644 buf_end_p = &ee->out_buf_end;
1645 }
1646 }
1647 else {
1648 rb_econv_elem_t *ee = &ec->elems[last_trans_index];
1649 buf_start_p = &ee->out_buf_start;
1650 data_start_p = &ee->out_data_start;
1651 data_end_p = &ee->out_data_end;
1652 buf_end_p = &ee->out_buf_end;
1653 tc = ec->elems[last_trans_index].tc;
1654 }
1655
1656 if (*buf_start_p == NULL) {
1657 unsigned char *buf = xmalloc(need);
1658 *buf_start_p = buf;
1659 *data_start_p = buf;
1660 *data_end_p = buf;
1661 *buf_end_p = buf+need;
1662 }
1663 else if ((size_t)(*buf_end_p - *data_end_p) < need) {
1664 MEMMOVE(*buf_start_p, *data_start_p, unsigned char, *data_end_p - *data_start_p);
1665 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1666 *data_start_p = *buf_start_p;
1667 if ((size_t)(*buf_end_p - *data_end_p) < need) {
1668 unsigned char *buf;
1669 size_t s = (*data_end_p - *buf_start_p) + need;
1670 if (s < need)
1671 goto fail;
1672 buf = xrealloc(*buf_start_p, s);
1673 *data_start_p = buf;
1674 *data_end_p = buf + (*data_end_p - *buf_start_p);
1675 *buf_start_p = buf;
1676 *buf_end_p = buf + s;
1677 }
1678 }
1679
1680 memcpy(*data_end_p, insert_str, insert_len);
1681 *data_end_p += insert_len;
1682 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1683 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1684 *data_end_p += tc->readagain_len;
1685 tc->readagain_len = 0;
1686 }
1687
1688 if (insert_str != str && insert_str != insert_buf)
1689 xfree((void*)insert_str);
1690 return 0;
1691
1692 fail:
1693 if (insert_str != str && insert_str != insert_buf)
1694 xfree((void*)insert_str);
1695 return -1;
1696 }
1697
1698 void
rb_econv_close(rb_econv_t * ec)1699 rb_econv_close(rb_econv_t *ec)
1700 {
1701 int i;
1702
1703 if (ec->replacement_allocated) {
1704 xfree((void *)ec->replacement_str);
1705 }
1706 for (i = 0; i < ec->num_trans; i++) {
1707 rb_transcoding_close(ec->elems[i].tc);
1708 if (ec->elems[i].out_buf_start)
1709 xfree(ec->elems[i].out_buf_start);
1710 }
1711 xfree(ec->in_buf_start);
1712 xfree(ec->elems);
1713 xfree(ec);
1714 }
1715
1716 size_t
rb_econv_memsize(rb_econv_t * ec)1717 rb_econv_memsize(rb_econv_t *ec)
1718 {
1719 size_t size = sizeof(rb_econv_t);
1720 int i;
1721
1722 if (ec->replacement_allocated) {
1723 size += ec->replacement_len;
1724 }
1725 for (i = 0; i < ec->num_trans; i++) {
1726 size += rb_transcoding_memsize(ec->elems[i].tc);
1727
1728 if (ec->elems[i].out_buf_start) {
1729 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1730 }
1731 }
1732 size += ec->in_buf_end - ec->in_buf_start;
1733 size += sizeof(rb_econv_elem_t) * ec->num_allocated;
1734
1735 return size;
1736 }
1737
1738 int
rb_econv_putbackable(rb_econv_t * ec)1739 rb_econv_putbackable(rb_econv_t *ec)
1740 {
1741 if (ec->num_trans == 0)
1742 return 0;
1743 #if SIZEOF_SIZE_T > SIZEOF_INT
1744 if (ec->elems[0].tc->readagain_len > INT_MAX) return INT_MAX;
1745 #endif
1746 return (int)ec->elems[0].tc->readagain_len;
1747 }
1748
1749 void
rb_econv_putback(rb_econv_t * ec,unsigned char * p,int n)1750 rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
1751 {
1752 rb_transcoding *tc;
1753 if (ec->num_trans == 0 || n == 0)
1754 return;
1755 tc = ec->elems[0].tc;
1756 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1757 tc->readagain_len -= n;
1758 }
1759
1760 struct asciicompat_encoding_t {
1761 const char *ascii_compat_name;
1762 const char *ascii_incompat_name;
1763 };
1764
1765 static int
asciicompat_encoding_i(st_data_t key,st_data_t val,st_data_t arg)1766 asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1767 {
1768 struct asciicompat_encoding_t *data = (struct asciicompat_encoding_t *)arg;
1769 transcoder_entry_t *entry = (transcoder_entry_t *)val;
1770 const rb_transcoder *tr;
1771
1772 if (DECORATOR_P(entry->sname, entry->dname))
1773 return ST_CONTINUE;
1774 tr = load_transcoder_entry(entry);
1775 if (tr && tr->asciicompat_type == asciicompat_decoder) {
1776 data->ascii_compat_name = tr->dst_encoding;
1777 return ST_STOP;
1778 }
1779 return ST_CONTINUE;
1780 }
1781
1782 const char *
rb_econv_asciicompat_encoding(const char * ascii_incompat_name)1783 rb_econv_asciicompat_encoding(const char *ascii_incompat_name)
1784 {
1785 st_data_t v;
1786 st_table *table2;
1787 struct asciicompat_encoding_t data;
1788
1789 if (!st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v))
1790 return NULL;
1791 table2 = (st_table *)v;
1792
1793 /*
1794 * Assumption:
1795 * There is at most one transcoder for
1796 * converting from ASCII incompatible encoding.
1797 *
1798 * For ISO-2022-JP, there is ISO-2022-JP -> stateless-ISO-2022-JP and no others.
1799 */
1800 if (table2->num_entries != 1)
1801 return NULL;
1802
1803 data.ascii_incompat_name = ascii_incompat_name;
1804 data.ascii_compat_name = NULL;
1805 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1806 return data.ascii_compat_name;
1807 }
1808
1809 VALUE
rb_econv_append(rb_econv_t * ec,const char * ss,long len,VALUE dst,int flags)1810 rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags)
1811 {
1812 unsigned const char *sp, *se;
1813 unsigned char *ds, *dp, *de;
1814 rb_econv_result_t res;
1815 int max_output;
1816
1817 if (NIL_P(dst)) {
1818 dst = rb_str_buf_new(len);
1819 if (ec->destination_encoding)
1820 rb_enc_associate(dst, ec->destination_encoding);
1821 }
1822
1823 if (ec->last_tc)
1824 max_output = ec->last_tc->transcoder->max_output;
1825 else
1826 max_output = 1;
1827
1828 do {
1829 long dlen = RSTRING_LEN(dst);
1830 if (rb_str_capacity(dst) - dlen < (size_t)len + max_output) {
1831 unsigned long new_capa = (unsigned long)dlen + len + max_output;
1832 if (LONG_MAX < new_capa)
1833 rb_raise(rb_eArgError, "too long string");
1834 rb_str_resize(dst, new_capa);
1835 rb_str_set_len(dst, dlen);
1836 }
1837 sp = (const unsigned char *)ss;
1838 se = sp + len;
1839 ds = (unsigned char *)RSTRING_PTR(dst);
1840 de = ds + rb_str_capacity(dst);
1841 dp = ds += dlen;
1842 res = rb_econv_convert(ec, &sp, se, &dp, de, flags);
1843 len -= (const char *)sp - ss;
1844 ss = (const char *)sp;
1845 rb_str_set_len(dst, dlen + (dp - ds));
1846 rb_econv_check_error(ec);
1847 } while (res == econv_destination_buffer_full);
1848
1849 return dst;
1850 }
1851
1852 VALUE
rb_econv_substr_append(rb_econv_t * ec,VALUE src,long off,long len,VALUE dst,int flags)1853 rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags)
1854 {
1855 src = rb_str_new_frozen(src);
1856 dst = rb_econv_append(ec, RSTRING_PTR(src) + off, len, dst, flags);
1857 RB_GC_GUARD(src);
1858 OBJ_INFECT_RAW(dst, src);
1859 return dst;
1860 }
1861
1862 VALUE
rb_econv_str_append(rb_econv_t * ec,VALUE src,VALUE dst,int flags)1863 rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
1864 {
1865 return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), dst, flags);
1866 }
1867
1868 VALUE
rb_econv_substr_convert(rb_econv_t * ec,VALUE src,long byteoff,long bytesize,int flags)1869 rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
1870 {
1871 return rb_econv_substr_append(ec, src, byteoff, bytesize, Qnil, flags);
1872 }
1873
1874 VALUE
rb_econv_str_convert(rb_econv_t * ec,VALUE src,int flags)1875 rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
1876 {
1877 return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), Qnil, flags);
1878 }
1879
1880 static int
rb_econv_add_converter(rb_econv_t * ec,const char * sname,const char * dname,int n)1881 rb_econv_add_converter(rb_econv_t *ec, const char *sname, const char *dname, int n)
1882 {
1883 transcoder_entry_t *entry;
1884 const rb_transcoder *tr;
1885
1886 if (ec->started != 0)
1887 return -1;
1888
1889 entry = get_transcoder_entry(sname, dname);
1890 if (!entry)
1891 return -1;
1892
1893 tr = load_transcoder_entry(entry);
1894 if (!tr) return -1;
1895
1896 return rb_econv_add_transcoder_at(ec, tr, n);
1897 }
1898
1899 static int
rb_econv_decorate_at(rb_econv_t * ec,const char * decorator_name,int n)1900 rb_econv_decorate_at(rb_econv_t *ec, const char *decorator_name, int n)
1901 {
1902 return rb_econv_add_converter(ec, "", decorator_name, n);
1903 }
1904
1905 int
rb_econv_decorate_at_first(rb_econv_t * ec,const char * decorator_name)1906 rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
1907 {
1908 const rb_transcoder *tr;
1909
1910 if (ec->num_trans == 0)
1911 return rb_econv_decorate_at(ec, decorator_name, 0);
1912
1913 tr = ec->elems[0].tc->transcoder;
1914
1915 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
1916 tr->asciicompat_type == asciicompat_decoder)
1917 return rb_econv_decorate_at(ec, decorator_name, 1);
1918
1919 return rb_econv_decorate_at(ec, decorator_name, 0);
1920 }
1921
1922 int
rb_econv_decorate_at_last(rb_econv_t * ec,const char * decorator_name)1923 rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
1924 {
1925 const rb_transcoder *tr;
1926
1927 if (ec->num_trans == 0)
1928 return rb_econv_decorate_at(ec, decorator_name, 0);
1929
1930 tr = ec->elems[ec->num_trans-1].tc->transcoder;
1931
1932 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
1933 tr->asciicompat_type == asciicompat_encoder)
1934 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
1935
1936 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
1937 }
1938
1939 void
rb_econv_binmode(rb_econv_t * ec)1940 rb_econv_binmode(rb_econv_t *ec)
1941 {
1942 const char *dname = 0;
1943
1944 switch (ec->flags & ECONV_NEWLINE_DECORATOR_MASK) {
1945 case ECONV_UNIVERSAL_NEWLINE_DECORATOR:
1946 dname = "universal_newline";
1947 break;
1948 case ECONV_CRLF_NEWLINE_DECORATOR:
1949 dname = "crlf_newline";
1950 break;
1951 case ECONV_CR_NEWLINE_DECORATOR:
1952 dname = "cr_newline";
1953 break;
1954 }
1955
1956 if (dname) {
1957 const rb_transcoder *transcoder = get_transcoder_entry("", dname)->transcoder;
1958 int num_trans = ec->num_trans;
1959 int i, j = 0;
1960
1961 for (i=0; i < num_trans; i++) {
1962 if (transcoder == ec->elems[i].tc->transcoder) {
1963 rb_transcoding_close(ec->elems[i].tc);
1964 xfree(ec->elems[i].out_buf_start);
1965 ec->num_trans--;
1966 }
1967 else
1968 ec->elems[j++] = ec->elems[i];
1969 }
1970 }
1971
1972 ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK;
1973 }
1974
1975 static VALUE
econv_description(const char * sname,const char * dname,int ecflags,VALUE mesg)1976 econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg)
1977 {
1978 int has_description = 0;
1979
1980 if (NIL_P(mesg))
1981 mesg = rb_str_new(NULL, 0);
1982
1983 if (*sname != '\0' || *dname != '\0') {
1984 if (*sname == '\0')
1985 rb_str_cat2(mesg, dname);
1986 else if (*dname == '\0')
1987 rb_str_cat2(mesg, sname);
1988 else
1989 rb_str_catf(mesg, "%s to %s", sname, dname);
1990 has_description = 1;
1991 }
1992
1993 if (ecflags & (ECONV_NEWLINE_DECORATOR_MASK|
1994 ECONV_XML_TEXT_DECORATOR|
1995 ECONV_XML_ATTR_CONTENT_DECORATOR|
1996 ECONV_XML_ATTR_QUOTE_DECORATOR)) {
1997 const char *pre = "";
1998 if (has_description)
1999 rb_str_cat2(mesg, " with ");
2000 if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR) {
2001 rb_str_cat2(mesg, pre); pre = ",";
2002 rb_str_cat2(mesg, "universal_newline");
2003 }
2004 if (ecflags & ECONV_CRLF_NEWLINE_DECORATOR) {
2005 rb_str_cat2(mesg, pre); pre = ",";
2006 rb_str_cat2(mesg, "crlf_newline");
2007 }
2008 if (ecflags & ECONV_CR_NEWLINE_DECORATOR) {
2009 rb_str_cat2(mesg, pre); pre = ",";
2010 rb_str_cat2(mesg, "cr_newline");
2011 }
2012 if (ecflags & ECONV_XML_TEXT_DECORATOR) {
2013 rb_str_cat2(mesg, pre); pre = ",";
2014 rb_str_cat2(mesg, "xml_text");
2015 }
2016 if (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR) {
2017 rb_str_cat2(mesg, pre); pre = ",";
2018 rb_str_cat2(mesg, "xml_attr_content");
2019 }
2020 if (ecflags & ECONV_XML_ATTR_QUOTE_DECORATOR) {
2021 rb_str_cat2(mesg, pre); pre = ",";
2022 rb_str_cat2(mesg, "xml_attr_quote");
2023 }
2024 has_description = 1;
2025 }
2026 if (!has_description) {
2027 rb_str_cat2(mesg, "no-conversion");
2028 }
2029
2030 return mesg;
2031 }
2032
2033 VALUE
rb_econv_open_exc(const char * sname,const char * dname,int ecflags)2034 rb_econv_open_exc(const char *sname, const char *dname, int ecflags)
2035 {
2036 VALUE mesg, exc;
2037 mesg = rb_str_new_cstr("code converter not found (");
2038 econv_description(sname, dname, ecflags, mesg);
2039 rb_str_cat2(mesg, ")");
2040 exc = rb_exc_new3(rb_eConverterNotFoundError, mesg);
2041 return exc;
2042 }
2043
2044 static VALUE
make_econv_exception(rb_econv_t * ec)2045 make_econv_exception(rb_econv_t *ec)
2046 {
2047 VALUE mesg, exc;
2048 if (ec->last_error.result == econv_invalid_byte_sequence ||
2049 ec->last_error.result == econv_incomplete_input) {
2050 const char *err = (const char *)ec->last_error.error_bytes_start;
2051 size_t error_len = ec->last_error.error_bytes_len;
2052 VALUE bytes = rb_str_new(err, error_len);
2053 VALUE dumped = rb_str_dump(bytes);
2054 size_t readagain_len = ec->last_error.readagain_len;
2055 VALUE bytes2 = Qnil;
2056 VALUE dumped2;
2057 int idx;
2058 if (ec->last_error.result == econv_incomplete_input) {
2059 mesg = rb_sprintf("incomplete %s on %s",
2060 StringValueCStr(dumped),
2061 ec->last_error.source_encoding);
2062 }
2063 else if (readagain_len) {
2064 bytes2 = rb_str_new(err+error_len, readagain_len);
2065 dumped2 = rb_str_dump(bytes2);
2066 mesg = rb_sprintf("%s followed by %s on %s",
2067 StringValueCStr(dumped),
2068 StringValueCStr(dumped2),
2069 ec->last_error.source_encoding);
2070 }
2071 else {
2072 mesg = rb_sprintf("%s on %s",
2073 StringValueCStr(dumped),
2074 ec->last_error.source_encoding);
2075 }
2076
2077 exc = rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2078 rb_ivar_set(exc, rb_intern("error_bytes"), bytes);
2079 rb_ivar_set(exc, rb_intern("readagain_bytes"), bytes2);
2080 rb_ivar_set(exc, rb_intern("incomplete_input"), ec->last_error.result == econv_incomplete_input ? Qtrue : Qfalse);
2081
2082 set_encs:
2083 rb_ivar_set(exc, rb_intern("source_encoding_name"), rb_str_new2(ec->last_error.source_encoding));
2084 rb_ivar_set(exc, rb_intern("destination_encoding_name"), rb_str_new2(ec->last_error.destination_encoding));
2085 idx = rb_enc_find_index(ec->last_error.source_encoding);
2086 if (0 <= idx)
2087 rb_ivar_set(exc, rb_intern("source_encoding"), rb_enc_from_encoding(rb_enc_from_index(idx)));
2088 idx = rb_enc_find_index(ec->last_error.destination_encoding);
2089 if (0 <= idx)
2090 rb_ivar_set(exc, rb_intern("destination_encoding"), rb_enc_from_encoding(rb_enc_from_index(idx)));
2091 return exc;
2092 }
2093 if (ec->last_error.result == econv_undefined_conversion) {
2094 VALUE bytes = rb_str_new((const char *)ec->last_error.error_bytes_start,
2095 ec->last_error.error_bytes_len);
2096 VALUE dumped = Qnil;
2097 int idx;
2098 if (strcmp(ec->last_error.source_encoding, "UTF-8") == 0) {
2099 rb_encoding *utf8 = rb_utf8_encoding();
2100 const char *start, *end;
2101 int n;
2102 start = (const char *)ec->last_error.error_bytes_start;
2103 end = start + ec->last_error.error_bytes_len;
2104 n = rb_enc_precise_mbclen(start, end, utf8);
2105 if (MBCLEN_CHARFOUND_P(n) &&
2106 (size_t)MBCLEN_CHARFOUND_LEN(n) == ec->last_error.error_bytes_len) {
2107 unsigned int cc = rb_enc_mbc_to_codepoint(start, end, utf8);
2108 dumped = rb_sprintf("U+%04X", cc);
2109 }
2110 }
2111 if (dumped == Qnil)
2112 dumped = rb_str_dump(bytes);
2113 if (strcmp(ec->last_error.source_encoding,
2114 ec->source_encoding_name) == 0 &&
2115 strcmp(ec->last_error.destination_encoding,
2116 ec->destination_encoding_name) == 0) {
2117 mesg = rb_sprintf("%s from %s to %s",
2118 StringValueCStr(dumped),
2119 ec->last_error.source_encoding,
2120 ec->last_error.destination_encoding);
2121 }
2122 else {
2123 int i;
2124 mesg = rb_sprintf("%s to %s in conversion from %s",
2125 StringValueCStr(dumped),
2126 ec->last_error.destination_encoding,
2127 ec->source_encoding_name);
2128 for (i = 0; i < ec->num_trans; i++) {
2129 const rb_transcoder *tr = ec->elems[i].tc->transcoder;
2130 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding))
2131 rb_str_catf(mesg, " to %s",
2132 ec->elems[i].tc->transcoder->dst_encoding);
2133 }
2134 }
2135 exc = rb_exc_new3(rb_eUndefinedConversionError, mesg);
2136 idx = rb_enc_find_index(ec->last_error.source_encoding);
2137 if (0 <= idx)
2138 rb_enc_associate_index(bytes, idx);
2139 rb_ivar_set(exc, rb_intern("error_char"), bytes);
2140 goto set_encs;
2141 }
2142 return Qnil;
2143 }
2144
2145 static void
more_output_buffer(VALUE destination,unsigned char * (* resize_destination)(VALUE,size_t,size_t),int max_output,unsigned char ** out_start_ptr,unsigned char ** out_pos,unsigned char ** out_stop_ptr)2146 more_output_buffer(
2147 VALUE destination,
2148 unsigned char *(*resize_destination)(VALUE, size_t, size_t),
2149 int max_output,
2150 unsigned char **out_start_ptr,
2151 unsigned char **out_pos,
2152 unsigned char **out_stop_ptr)
2153 {
2154 size_t len = (*out_pos - *out_start_ptr);
2155 size_t new_len = (len + max_output) * 2;
2156 *out_start_ptr = resize_destination(destination, len, new_len);
2157 *out_pos = *out_start_ptr + len;
2158 *out_stop_ptr = *out_start_ptr + new_len;
2159 }
2160
2161 static int
make_replacement(rb_econv_t * ec)2162 make_replacement(rb_econv_t *ec)
2163 {
2164 rb_transcoding *tc;
2165 const rb_transcoder *tr;
2166 const unsigned char *replacement;
2167 const char *repl_enc;
2168 const char *ins_enc;
2169 size_t len;
2170
2171 if (ec->replacement_str)
2172 return 0;
2173
2174 ins_enc = rb_econv_encoding_to_insert_output(ec);
2175
2176 tc = ec->last_tc;
2177 if (*ins_enc) {
2178 tr = tc->transcoder;
2179 rb_enc_find(tr->dst_encoding);
2180 replacement = (const unsigned char *)get_replacement_character(ins_enc, &len, &repl_enc);
2181 }
2182 else {
2183 replacement = (unsigned char *)"?";
2184 len = 1;
2185 repl_enc = "";
2186 }
2187
2188 ec->replacement_str = replacement;
2189 ec->replacement_len = len;
2190 ec->replacement_enc = repl_enc;
2191 ec->replacement_allocated = 0;
2192 return 0;
2193 }
2194
2195 int
rb_econv_set_replacement(rb_econv_t * ec,const unsigned char * str,size_t len,const char * encname)2196 rb_econv_set_replacement(rb_econv_t *ec,
2197 const unsigned char *str, size_t len, const char *encname)
2198 {
2199 unsigned char *str2;
2200 size_t len2;
2201 const char *encname2;
2202
2203 encname2 = rb_econv_encoding_to_insert_output(ec);
2204
2205 if (!*encname2 || encoding_equal(encname, encname2)) {
2206 str2 = xmalloc(len);
2207 MEMCPY(str2, str, unsigned char, len); /* xxx: str may be invalid */
2208 len2 = len;
2209 encname2 = encname;
2210 }
2211 else {
2212 str2 = allocate_converted_string(encname, encname2, str, len, NULL, 0, &len2);
2213 if (!str2)
2214 return -1;
2215 }
2216
2217 if (ec->replacement_allocated) {
2218 xfree((void *)ec->replacement_str);
2219 }
2220 ec->replacement_allocated = 1;
2221 ec->replacement_str = str2;
2222 ec->replacement_len = len2;
2223 ec->replacement_enc = encname2;
2224 return 0;
2225 }
2226
2227 static int
output_replacement_character(rb_econv_t * ec)2228 output_replacement_character(rb_econv_t *ec)
2229 {
2230 int ret;
2231
2232 if (make_replacement(ec) == -1)
2233 return -1;
2234
2235 ret = rb_econv_insert_output(ec, ec->replacement_str, ec->replacement_len, ec->replacement_enc);
2236 if (ret == -1)
2237 return -1;
2238
2239 return 0;
2240 }
2241
2242 #if 1
2243 #define hash_fallback rb_hash_aref
2244
2245 static VALUE
proc_fallback(VALUE fallback,VALUE c)2246 proc_fallback(VALUE fallback, VALUE c)
2247 {
2248 return rb_proc_call(fallback, rb_ary_new4(1, &c));
2249 }
2250
2251 static VALUE
method_fallback(VALUE fallback,VALUE c)2252 method_fallback(VALUE fallback, VALUE c)
2253 {
2254 return rb_method_call(1, &c, fallback);
2255 }
2256
2257 static VALUE
aref_fallback(VALUE fallback,VALUE c)2258 aref_fallback(VALUE fallback, VALUE c)
2259 {
2260 return rb_funcallv_public(fallback, idAREF, 1, &c);
2261 }
2262
2263 static void
transcode_loop(const unsigned char ** in_pos,unsigned char ** out_pos,const unsigned char * in_stop,unsigned char * out_stop,VALUE destination,unsigned char * (* resize_destination)(VALUE,size_t,size_t),const char * src_encoding,const char * dst_encoding,int ecflags,VALUE ecopts)2264 transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
2265 const unsigned char *in_stop, unsigned char *out_stop,
2266 VALUE destination,
2267 unsigned char *(*resize_destination)(VALUE, size_t, size_t),
2268 const char *src_encoding,
2269 const char *dst_encoding,
2270 int ecflags,
2271 VALUE ecopts)
2272 {
2273 rb_econv_t *ec;
2274 rb_transcoding *last_tc;
2275 rb_econv_result_t ret;
2276 unsigned char *out_start = *out_pos;
2277 int max_output;
2278 VALUE exc;
2279 VALUE fallback = Qnil;
2280 VALUE (*fallback_func)(VALUE, VALUE) = 0;
2281
2282 ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts);
2283 if (!ec)
2284 rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags));
2285
2286 if (!NIL_P(ecopts) && RB_TYPE_P(ecopts, T_HASH)) {
2287 fallback = rb_hash_aref(ecopts, sym_fallback);
2288 if (RB_TYPE_P(fallback, T_HASH)) {
2289 fallback_func = hash_fallback;
2290 }
2291 else if (rb_obj_is_proc(fallback)) {
2292 fallback_func = proc_fallback;
2293 }
2294 else if (rb_obj_is_method(fallback)) {
2295 fallback_func = method_fallback;
2296 }
2297 else {
2298 fallback_func = aref_fallback;
2299 }
2300 }
2301 last_tc = ec->last_tc;
2302 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2303
2304 resume:
2305 ret = rb_econv_convert(ec, in_pos, in_stop, out_pos, out_stop, 0);
2306
2307 if (!NIL_P(fallback) && ret == econv_undefined_conversion) {
2308 VALUE rep = rb_enc_str_new(
2309 (const char *)ec->last_error.error_bytes_start,
2310 ec->last_error.error_bytes_len,
2311 rb_enc_find(ec->last_error.source_encoding));
2312 rep = (*fallback_func)(fallback, rep);
2313 if (rep != Qundef && !NIL_P(rep)) {
2314 StringValue(rep);
2315 ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep),
2316 RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
2317 if ((int)ret == -1) {
2318 rb_raise(rb_eArgError, "too big fallback string");
2319 }
2320 goto resume;
2321 }
2322 }
2323
2324 if (ret == econv_invalid_byte_sequence ||
2325 ret == econv_incomplete_input ||
2326 ret == econv_undefined_conversion) {
2327 exc = make_econv_exception(ec);
2328 rb_econv_close(ec);
2329 rb_exc_raise(exc);
2330 }
2331
2332 if (ret == econv_destination_buffer_full) {
2333 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2334 goto resume;
2335 }
2336
2337 rb_econv_close(ec);
2338 return;
2339 }
2340 #else
2341 /* sample transcode_loop implementation in byte-by-byte stream style */
2342 static void
transcode_loop(const unsigned char ** in_pos,unsigned char ** out_pos,const unsigned char * in_stop,unsigned char * out_stop,VALUE destination,unsigned char * (* resize_destination)(VALUE,size_t,size_t),const char * src_encoding,const char * dst_encoding,int ecflags,VALUE ecopts)2343 transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
2344 const unsigned char *in_stop, unsigned char *out_stop,
2345 VALUE destination,
2346 unsigned char *(*resize_destination)(VALUE, size_t, size_t),
2347 const char *src_encoding,
2348 const char *dst_encoding,
2349 int ecflags,
2350 VALUE ecopts)
2351 {
2352 rb_econv_t *ec;
2353 rb_transcoding *last_tc;
2354 rb_econv_result_t ret;
2355 unsigned char *out_start = *out_pos;
2356 const unsigned char *ptr;
2357 int max_output;
2358 VALUE exc;
2359
2360 ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts);
2361 if (!ec)
2362 rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags));
2363
2364 last_tc = ec->last_tc;
2365 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2366
2367 ret = econv_source_buffer_empty;
2368 ptr = *in_pos;
2369 while (ret != econv_finished) {
2370 unsigned char input_byte;
2371 const unsigned char *p = &input_byte;
2372
2373 if (ret == econv_source_buffer_empty) {
2374 if (ptr < in_stop) {
2375 input_byte = *ptr;
2376 ret = rb_econv_convert(ec, &p, p+1, out_pos, out_stop, ECONV_PARTIAL_INPUT);
2377 }
2378 else {
2379 ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, 0);
2380 }
2381 }
2382 else {
2383 ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, ECONV_PARTIAL_INPUT);
2384 }
2385 if (&input_byte != p)
2386 ptr += p - &input_byte;
2387 switch (ret) {
2388 case econv_invalid_byte_sequence:
2389 case econv_incomplete_input:
2390 case econv_undefined_conversion:
2391 exc = make_econv_exception(ec);
2392 rb_econv_close(ec);
2393 rb_exc_raise(exc);
2394 break;
2395
2396 case econv_destination_buffer_full:
2397 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2398 break;
2399
2400 case econv_source_buffer_empty:
2401 break;
2402
2403 case econv_finished:
2404 break;
2405 }
2406 }
2407 rb_econv_close(ec);
2408 *in_pos = in_stop;
2409 return;
2410 }
2411 #endif
2412
2413
2414 /*
2415 * String-specific code
2416 */
2417
2418 static unsigned char *
str_transcoding_resize(VALUE destination,size_t len,size_t new_len)2419 str_transcoding_resize(VALUE destination, size_t len, size_t new_len)
2420 {
2421 rb_str_resize(destination, new_len);
2422 return (unsigned char *)RSTRING_PTR(destination);
2423 }
2424
2425 static int
econv_opts(VALUE opt,int ecflags)2426 econv_opts(VALUE opt, int ecflags)
2427 {
2428 VALUE v;
2429
2430 v = rb_hash_aref(opt, sym_invalid);
2431 if (NIL_P(v)) {
2432 }
2433 else if (v==sym_replace) {
2434 ecflags |= ECONV_INVALID_REPLACE;
2435 }
2436 else {
2437 rb_raise(rb_eArgError, "unknown value for invalid character option");
2438 }
2439
2440 v = rb_hash_aref(opt, sym_undef);
2441 if (NIL_P(v)) {
2442 }
2443 else if (v==sym_replace) {
2444 ecflags |= ECONV_UNDEF_REPLACE;
2445 }
2446 else {
2447 rb_raise(rb_eArgError, "unknown value for undefined character option");
2448 }
2449
2450 v = rb_hash_aref(opt, sym_replace);
2451 if (!NIL_P(v) && !(ecflags & ECONV_INVALID_REPLACE)) {
2452 ecflags |= ECONV_UNDEF_REPLACE;
2453 }
2454
2455 v = rb_hash_aref(opt, sym_xml);
2456 if (!NIL_P(v)) {
2457 if (v==sym_text) {
2458 ecflags |= ECONV_XML_TEXT_DECORATOR|ECONV_UNDEF_HEX_CHARREF;
2459 }
2460 else if (v==sym_attr) {
2461 ecflags |= ECONV_XML_ATTR_CONTENT_DECORATOR|ECONV_XML_ATTR_QUOTE_DECORATOR|ECONV_UNDEF_HEX_CHARREF;
2462 }
2463 else if (RB_TYPE_P(v, T_SYMBOL)) {
2464 rb_raise(rb_eArgError, "unexpected value for xml option: %"PRIsVALUE, rb_sym2str(v));
2465 }
2466 else {
2467 rb_raise(rb_eArgError, "unexpected value for xml option");
2468 }
2469 }
2470
2471 #ifdef ENABLE_ECONV_NEWLINE_OPTION
2472 v = rb_hash_aref(opt, sym_newline);
2473 if (!NIL_P(v)) {
2474 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2475 if (v == sym_universal) {
2476 ecflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR;
2477 }
2478 else if (v == sym_crlf) {
2479 ecflags |= ECONV_CRLF_NEWLINE_DECORATOR;
2480 }
2481 else if (v == sym_cr) {
2482 ecflags |= ECONV_CR_NEWLINE_DECORATOR;
2483 }
2484 else if (v == sym_lf) {
2485 /* ecflags |= ECONV_LF_NEWLINE_DECORATOR; */
2486 }
2487 else if (SYMBOL_P(v)) {
2488 rb_raise(rb_eArgError, "unexpected value for newline option: %"PRIsVALUE,
2489 rb_sym2str(v));
2490 }
2491 else {
2492 rb_raise(rb_eArgError, "unexpected value for newline option");
2493 }
2494 }
2495 else
2496 #endif
2497 {
2498 int setflags = 0, newlineflag = 0;
2499
2500 v = rb_hash_aref(opt, sym_universal_newline);
2501 if (RTEST(v))
2502 setflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR;
2503 newlineflag |= !NIL_P(v);
2504
2505 v = rb_hash_aref(opt, sym_crlf_newline);
2506 if (RTEST(v))
2507 setflags |= ECONV_CRLF_NEWLINE_DECORATOR;
2508 newlineflag |= !NIL_P(v);
2509
2510 v = rb_hash_aref(opt, sym_cr_newline);
2511 if (RTEST(v))
2512 setflags |= ECONV_CR_NEWLINE_DECORATOR;
2513 newlineflag |= !NIL_P(v);
2514
2515 if (newlineflag) {
2516 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2517 ecflags |= setflags;
2518 }
2519 }
2520
2521 return ecflags;
2522 }
2523
2524 int
rb_econv_prepare_options(VALUE opthash,VALUE * opts,int ecflags)2525 rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags)
2526 {
2527 VALUE newhash = Qnil;
2528 VALUE v;
2529
2530 if (NIL_P(opthash)) {
2531 *opts = Qnil;
2532 return ecflags;
2533 }
2534 ecflags = econv_opts(opthash, ecflags);
2535
2536 v = rb_hash_aref(opthash, sym_replace);
2537 if (!NIL_P(v)) {
2538 StringValue(v);
2539 if (rb_enc_str_coderange(v) == ENC_CODERANGE_BROKEN) {
2540 VALUE dumped = rb_str_dump(v);
2541 rb_raise(rb_eArgError, "replacement string is broken: %s as %s",
2542 StringValueCStr(dumped),
2543 rb_enc_name(rb_enc_get(v)));
2544 }
2545 v = rb_str_new_frozen(v);
2546 newhash = rb_hash_new();
2547 rb_hash_aset(newhash, sym_replace, v);
2548 }
2549
2550 v = rb_hash_aref(opthash, sym_fallback);
2551 if (!NIL_P(v)) {
2552 VALUE h = rb_check_hash_type(v);
2553 if (NIL_P(h)
2554 ? (rb_obj_is_proc(v) || rb_obj_is_method(v) || rb_respond_to(v, idAREF))
2555 : (v = h, 1)) {
2556 if (NIL_P(newhash))
2557 newhash = rb_hash_new();
2558 rb_hash_aset(newhash, sym_fallback, v);
2559 }
2560 }
2561
2562 if (!NIL_P(newhash))
2563 rb_hash_freeze(newhash);
2564 *opts = newhash;
2565
2566 return ecflags;
2567 }
2568
2569 int
rb_econv_prepare_opts(VALUE opthash,VALUE * opts)2570 rb_econv_prepare_opts(VALUE opthash, VALUE *opts)
2571 {
2572 return rb_econv_prepare_options(opthash, opts, 0);
2573 }
2574
2575 rb_econv_t *
rb_econv_open_opts(const char * source_encoding,const char * destination_encoding,int ecflags,VALUE opthash)2576 rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE opthash)
2577 {
2578 rb_econv_t *ec;
2579 VALUE replacement;
2580
2581 if (NIL_P(opthash)) {
2582 replacement = Qnil;
2583 }
2584 else {
2585 if (!RB_TYPE_P(opthash, T_HASH) || !OBJ_FROZEN(opthash))
2586 rb_bug("rb_econv_open_opts called with invalid opthash");
2587 replacement = rb_hash_aref(opthash, sym_replace);
2588 }
2589
2590 ec = rb_econv_open(source_encoding, destination_encoding, ecflags);
2591 if (!ec)
2592 return ec;
2593
2594 if (!NIL_P(replacement)) {
2595 int ret;
2596 rb_encoding *enc = rb_enc_get(replacement);
2597
2598 ret = rb_econv_set_replacement(ec,
2599 (const unsigned char *)RSTRING_PTR(replacement),
2600 RSTRING_LEN(replacement),
2601 rb_enc_name(enc));
2602 if (ret == -1) {
2603 rb_econv_close(ec);
2604 return NULL;
2605 }
2606 }
2607 return ec;
2608 }
2609
2610 static int
enc_arg(VALUE * arg,const char ** name_p,rb_encoding ** enc_p)2611 enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p)
2612 {
2613 rb_encoding *enc;
2614 const char *n;
2615 int encidx;
2616 VALUE encval;
2617
2618 if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) ||
2619 !(enc = rb_enc_from_index(encidx))) {
2620 enc = NULL;
2621 encidx = 0;
2622 n = StringValueCStr(*arg);
2623 }
2624 else {
2625 n = rb_enc_name(enc);
2626 }
2627
2628 *name_p = n;
2629 *enc_p = enc;
2630
2631 return encidx;
2632 }
2633
2634 static int
str_transcode_enc_args(VALUE str,VALUE * arg1,VALUE * arg2,const char ** sname_p,rb_encoding ** senc_p,const char ** dname_p,rb_encoding ** denc_p)2635 str_transcode_enc_args(VALUE str, VALUE *arg1, VALUE *arg2,
2636 const char **sname_p, rb_encoding **senc_p,
2637 const char **dname_p, rb_encoding **denc_p)
2638 {
2639 rb_encoding *senc, *denc;
2640 const char *sname, *dname;
2641 int sencidx, dencidx;
2642
2643 dencidx = enc_arg(arg1, &dname, &denc);
2644
2645 if (NIL_P(*arg2)) {
2646 sencidx = rb_enc_get_index(str);
2647 senc = rb_enc_from_index(sencidx);
2648 sname = rb_enc_name(senc);
2649 }
2650 else {
2651 sencidx = enc_arg(arg2, &sname, &senc);
2652 }
2653
2654 *sname_p = sname;
2655 *senc_p = senc;
2656 *dname_p = dname;
2657 *denc_p = denc;
2658 return dencidx;
2659 }
2660
2661 static int
str_transcode0(int argc,VALUE * argv,VALUE * self,int ecflags,VALUE ecopts)2662 str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
2663 {
2664 VALUE dest;
2665 VALUE str = *self;
2666 VALUE arg1, arg2;
2667 long blen, slen;
2668 unsigned char *buf, *bp, *sp;
2669 const unsigned char *fromp;
2670 rb_encoding *senc, *denc;
2671 const char *sname, *dname;
2672 int dencidx;
2673 int explicitly_invalid_replace = TRUE;
2674
2675 rb_check_arity(argc, 0, 2);
2676
2677 if (argc == 0) {
2678 arg1 = rb_enc_default_internal();
2679 if (NIL_P(arg1)) {
2680 if (!ecflags) return -1;
2681 arg1 = rb_obj_encoding(str);
2682 }
2683 if (!(ecflags & ECONV_INVALID_MASK)) {
2684 explicitly_invalid_replace = FALSE;
2685 }
2686 ecflags |= ECONV_INVALID_REPLACE | ECONV_UNDEF_REPLACE;
2687 }
2688 else {
2689 arg1 = argv[0];
2690 }
2691 arg2 = argc<=1 ? Qnil : argv[1];
2692 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2693
2694 if ((ecflags & (ECONV_NEWLINE_DECORATOR_MASK|
2695 ECONV_XML_TEXT_DECORATOR|
2696 ECONV_XML_ATTR_CONTENT_DECORATOR|
2697 ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) {
2698 if (senc && senc == denc) {
2699 if ((ecflags & ECONV_INVALID_MASK) && explicitly_invalid_replace) {
2700 VALUE rep = Qnil;
2701 if (!NIL_P(ecopts)) {
2702 rep = rb_hash_aref(ecopts, sym_replace);
2703 }
2704 dest = rb_enc_str_scrub(senc, str, rep);
2705 if (NIL_P(dest)) dest = str;
2706 *self = dest;
2707 return dencidx;
2708 }
2709 return NIL_P(arg2) ? -1 : dencidx;
2710 }
2711 if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
2712 if (rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT) {
2713 return dencidx;
2714 }
2715 }
2716 if (encoding_equal(sname, dname)) {
2717 return NIL_P(arg2) ? -1 : dencidx;
2718 }
2719 }
2720 else {
2721 if (encoding_equal(sname, dname)) {
2722 sname = "";
2723 dname = "";
2724 }
2725 }
2726
2727 fromp = sp = (unsigned char *)RSTRING_PTR(str);
2728 slen = RSTRING_LEN(str);
2729 blen = slen + 30; /* len + margin */
2730 dest = rb_str_tmp_new(blen);
2731 bp = (unsigned char *)RSTRING_PTR(dest);
2732
2733 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts);
2734 if (fromp != sp+slen) {
2735 rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp);
2736 }
2737 buf = (unsigned char *)RSTRING_PTR(dest);
2738 *bp = '\0';
2739 rb_str_set_len(dest, bp - buf);
2740
2741 /* set encoding */
2742 if (!denc) {
2743 dencidx = rb_define_dummy_encoding(dname);
2744 RB_GC_GUARD(arg1);
2745 RB_GC_GUARD(arg2);
2746 }
2747 *self = dest;
2748
2749 return dencidx;
2750 }
2751
2752 static int
str_transcode(int argc,VALUE * argv,VALUE * self)2753 str_transcode(int argc, VALUE *argv, VALUE *self)
2754 {
2755 VALUE opt;
2756 int ecflags = 0;
2757 VALUE ecopts = Qnil;
2758
2759 argc = rb_scan_args(argc, argv, "02:", NULL, NULL, &opt);
2760 if (!NIL_P(opt)) {
2761 ecflags = rb_econv_prepare_opts(opt, &ecopts);
2762 }
2763 return str_transcode0(argc, argv, self, ecflags, ecopts);
2764 }
2765
2766 static inline VALUE
str_encode_associate(VALUE str,int encidx)2767 str_encode_associate(VALUE str, int encidx)
2768 {
2769 int cr = 0;
2770
2771 rb_enc_associate_index(str, encidx);
2772
2773 /* transcoded string never be broken. */
2774 if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
2775 rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr);
2776 }
2777 else {
2778 cr = ENC_CODERANGE_VALID;
2779 }
2780 ENC_CODERANGE_SET(str, cr);
2781 return str;
2782 }
2783
2784 /*
2785 * call-seq:
2786 * str.encode!(encoding [, options] ) -> str
2787 * str.encode!(dst_encoding, src_encoding [, options] ) -> str
2788 *
2789 * The first form transcodes the contents of <i>str</i> from
2790 * str.encoding to +encoding+.
2791 * The second form transcodes the contents of <i>str</i> from
2792 * src_encoding to dst_encoding.
2793 * The options Hash gives details for conversion. See String#encode
2794 * for details.
2795 * Returns the string even if no changes were made.
2796 */
2797
2798 static VALUE
str_encode_bang(int argc,VALUE * argv,VALUE str)2799 str_encode_bang(int argc, VALUE *argv, VALUE str)
2800 {
2801 VALUE newstr;
2802 int encidx;
2803
2804 rb_check_frozen(str);
2805
2806 newstr = str;
2807 encidx = str_transcode(argc, argv, &newstr);
2808
2809 if (encidx < 0) return str;
2810 if (newstr == str) {
2811 rb_enc_associate_index(str, encidx);
2812 return str;
2813 }
2814 rb_str_shared_replace(str, newstr);
2815 return str_encode_associate(str, encidx);
2816 }
2817
2818 static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx);
2819
2820 /*
2821 * call-seq:
2822 * str.encode(encoding [, options] ) -> str
2823 * str.encode(dst_encoding, src_encoding [, options] ) -> str
2824 * str.encode([options]) -> str
2825 *
2826 * The first form returns a copy of +str+ transcoded
2827 * to encoding +encoding+.
2828 * The second form returns a copy of +str+ transcoded
2829 * from src_encoding to dst_encoding.
2830 * The last form returns a copy of +str+ transcoded to
2831 * <tt>Encoding.default_internal</tt>.
2832 *
2833 * By default, the first and second form raise
2834 * Encoding::UndefinedConversionError for characters that are
2835 * undefined in the destination encoding, and
2836 * Encoding::InvalidByteSequenceError for invalid byte sequences
2837 * in the source encoding. The last form by default does not raise
2838 * exceptions but uses replacement strings.
2839 *
2840 * The +options+ Hash gives details for conversion and can have the following
2841 * keys:
2842 *
2843 * :invalid ::
2844 * If the value is +:replace+, #encode replaces invalid byte sequences in
2845 * +str+ with the replacement character. The default is to raise the
2846 * Encoding::InvalidByteSequenceError exception
2847 * :undef ::
2848 * If the value is +:replace+, #encode replaces characters which are
2849 * undefined in the destination encoding with the replacement character.
2850 * The default is to raise the Encoding::UndefinedConversionError.
2851 * :replace ::
2852 * Sets the replacement string to the given value. The default replacement
2853 * string is "\uFFFD" for Unicode encoding forms, and "?" otherwise.
2854 * :fallback ::
2855 * Sets the replacement string by the given object for undefined
2856 * character. The object should be a Hash, a Proc, a Method, or an
2857 * object which has [] method.
2858 * Its key is an undefined character encoded in the source encoding
2859 * of current transcoder. Its value can be any encoding until it
2860 * can be converted into the destination encoding of the transcoder.
2861 * :xml ::
2862 * The value must be +:text+ or +:attr+.
2863 * If the value is +:text+ #encode replaces undefined characters with their
2864 * (upper-case hexadecimal) numeric character references. '&', '<', and '>'
2865 * are converted to "&", "<", and ">", respectively.
2866 * If the value is +:attr+, #encode also quotes the replacement result
2867 * (using '"'), and replaces '"' with """.
2868 * :cr_newline ::
2869 * Replaces LF ("\n") with CR ("\r") if value is true.
2870 * :crlf_newline ::
2871 * Replaces LF ("\n") with CRLF ("\r\n") if value is true.
2872 * :universal_newline ::
2873 * Replaces CRLF ("\r\n") and CR ("\r") with LF ("\n") if value is true.
2874 */
2875
2876 static VALUE
str_encode(int argc,VALUE * argv,VALUE str)2877 str_encode(int argc, VALUE *argv, VALUE str)
2878 {
2879 VALUE newstr = str;
2880 int encidx = str_transcode(argc, argv, &newstr);
2881 return encoded_dup(newstr, str, encidx);
2882 }
2883
2884 VALUE
rb_str_encode(VALUE str,VALUE to,int ecflags,VALUE ecopts)2885 rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
2886 {
2887 int argc = 1;
2888 VALUE *argv = &to;
2889 VALUE newstr = str;
2890 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2891 return encoded_dup(newstr, str, encidx);
2892 }
2893
2894 static VALUE
encoded_dup(VALUE newstr,VALUE str,int encidx)2895 encoded_dup(VALUE newstr, VALUE str, int encidx)
2896 {
2897 if (encidx < 0) return rb_str_dup(str);
2898 if (newstr == str) {
2899 newstr = rb_str_dup(str);
2900 rb_enc_associate_index(newstr, encidx);
2901 return newstr;
2902 }
2903 else {
2904 RBASIC_SET_CLASS(newstr, rb_obj_class(str));
2905 }
2906 return str_encode_associate(newstr, encidx);
2907 }
2908
2909 /*
2910 * Document-class: Encoding::Converter
2911 *
2912 * Encoding conversion class.
2913 */
2914 static void
econv_free(void * ptr)2915 econv_free(void *ptr)
2916 {
2917 rb_econv_t *ec = ptr;
2918 rb_econv_close(ec);
2919 }
2920
2921 static size_t
econv_memsize(const void * ptr)2922 econv_memsize(const void *ptr)
2923 {
2924 return sizeof(rb_econv_t);
2925 }
2926
2927 static const rb_data_type_t econv_data_type = {
2928 "econv",
2929 {NULL, econv_free, econv_memsize,},
2930 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
2931 };
2932
2933 static VALUE
econv_s_allocate(VALUE klass)2934 econv_s_allocate(VALUE klass)
2935 {
2936 return TypedData_Wrap_Struct(klass, &econv_data_type, NULL);
2937 }
2938
2939 static rb_encoding *
make_dummy_encoding(const char * name)2940 make_dummy_encoding(const char *name)
2941 {
2942 rb_encoding *enc;
2943 int idx;
2944 idx = rb_define_dummy_encoding(name);
2945 enc = rb_enc_from_index(idx);
2946 return enc;
2947 }
2948
2949 static rb_encoding *
make_encoding(const char * name)2950 make_encoding(const char *name)
2951 {
2952 rb_encoding *enc;
2953 enc = rb_enc_find(name);
2954 if (!enc)
2955 enc = make_dummy_encoding(name);
2956 return enc;
2957 }
2958
2959 static VALUE
make_encobj(const char * name)2960 make_encobj(const char *name)
2961 {
2962 return rb_enc_from_encoding(make_encoding(name));
2963 }
2964
2965 /*
2966 * call-seq:
2967 * Encoding::Converter.asciicompat_encoding(string) -> encoding or nil
2968 * Encoding::Converter.asciicompat_encoding(encoding) -> encoding or nil
2969 *
2970 * Returns the corresponding ASCII compatible encoding.
2971 *
2972 * Returns nil if the argument is an ASCII compatible encoding.
2973 *
2974 * "corresponding ASCII compatible encoding" is an ASCII compatible encoding which
2975 * can represents exactly the same characters as the given ASCII incompatible encoding.
2976 * So, no conversion undefined error occurs when converting between the two encodings.
2977 *
2978 * Encoding::Converter.asciicompat_encoding("ISO-2022-JP") #=> #<Encoding:stateless-ISO-2022-JP>
2979 * Encoding::Converter.asciicompat_encoding("UTF-16BE") #=> #<Encoding:UTF-8>
2980 * Encoding::Converter.asciicompat_encoding("UTF-8") #=> nil
2981 *
2982 */
2983 static VALUE
econv_s_asciicompat_encoding(VALUE klass,VALUE arg)2984 econv_s_asciicompat_encoding(VALUE klass, VALUE arg)
2985 {
2986 const char *arg_name, *result_name;
2987 rb_encoding *arg_enc, *result_enc;
2988
2989 enc_arg(&arg, &arg_name, &arg_enc);
2990
2991 result_name = rb_econv_asciicompat_encoding(arg_name);
2992
2993 if (result_name == NULL)
2994 return Qnil;
2995
2996 result_enc = make_encoding(result_name);
2997
2998 return rb_enc_from_encoding(result_enc);
2999 }
3000
3001 static void
econv_args(int argc,VALUE * argv,VALUE * snamev_p,VALUE * dnamev_p,const char ** sname_p,const char ** dname_p,rb_encoding ** senc_p,rb_encoding ** denc_p,int * ecflags_p,VALUE * ecopts_p)3002 econv_args(int argc, VALUE *argv,
3003 VALUE *snamev_p, VALUE *dnamev_p,
3004 const char **sname_p, const char **dname_p,
3005 rb_encoding **senc_p, rb_encoding **denc_p,
3006 int *ecflags_p,
3007 VALUE *ecopts_p)
3008 {
3009 VALUE opt, flags_v, ecopts;
3010 int sidx, didx;
3011 const char *sname, *dname;
3012 rb_encoding *senc, *denc;
3013 int ecflags;
3014
3015 argc = rb_scan_args(argc, argv, "21:", snamev_p, dnamev_p, &flags_v, &opt);
3016
3017 if (!NIL_P(flags_v)) {
3018 if (!NIL_P(opt)) {
3019 rb_error_arity(argc + 1, 2, 3);
3020 }
3021 ecflags = NUM2INT(rb_to_int(flags_v));
3022 ecopts = Qnil;
3023 }
3024 else if (!NIL_P(opt)) {
3025 ecflags = rb_econv_prepare_opts(opt, &ecopts);
3026 }
3027 else {
3028 ecflags = 0;
3029 ecopts = Qnil;
3030 }
3031
3032 senc = NULL;
3033 sidx = rb_to_encoding_index(*snamev_p);
3034 if (0 <= sidx) {
3035 senc = rb_enc_from_index(sidx);
3036 }
3037 else {
3038 StringValue(*snamev_p);
3039 }
3040
3041 denc = NULL;
3042 didx = rb_to_encoding_index(*dnamev_p);
3043 if (0 <= didx) {
3044 denc = rb_enc_from_index(didx);
3045 }
3046 else {
3047 StringValue(*dnamev_p);
3048 }
3049
3050 sname = senc ? rb_enc_name(senc) : StringValueCStr(*snamev_p);
3051 dname = denc ? rb_enc_name(denc) : StringValueCStr(*dnamev_p);
3052
3053 *sname_p = sname;
3054 *dname_p = dname;
3055 *senc_p = senc;
3056 *denc_p = denc;
3057 *ecflags_p = ecflags;
3058 *ecopts_p = ecopts;
3059 }
3060
3061 static int
decorate_convpath(VALUE convpath,int ecflags)3062 decorate_convpath(VALUE convpath, int ecflags)
3063 {
3064 int num_decorators;
3065 const char *decorators[MAX_ECFLAGS_DECORATORS];
3066 int i;
3067 int n, len;
3068
3069 num_decorators = decorator_names(ecflags, decorators);
3070 if (num_decorators == -1)
3071 return -1;
3072
3073 len = n = RARRAY_LENINT(convpath);
3074 if (n != 0) {
3075 VALUE pair = RARRAY_AREF(convpath, n-1);
3076 if (RB_TYPE_P(pair, T_ARRAY)) {
3077 const char *sname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 0)));
3078 const char *dname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 1)));
3079 transcoder_entry_t *entry = get_transcoder_entry(sname, dname);
3080 const rb_transcoder *tr = load_transcoder_entry(entry);
3081 if (!tr)
3082 return -1;
3083 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
3084 tr->asciicompat_type == asciicompat_encoder) {
3085 n--;
3086 rb_ary_store(convpath, len + num_decorators - 1, pair);
3087 }
3088 }
3089 else {
3090 rb_ary_store(convpath, len + num_decorators - 1, pair);
3091 }
3092 }
3093
3094 for (i = 0; i < num_decorators; i++)
3095 rb_ary_store(convpath, n + i, rb_str_new_cstr(decorators[i]));
3096
3097 return 0;
3098 }
3099
3100 static void
search_convpath_i(const char * sname,const char * dname,int depth,void * arg)3101 search_convpath_i(const char *sname, const char *dname, int depth, void *arg)
3102 {
3103 VALUE *ary_p = arg;
3104 VALUE v;
3105
3106 if (*ary_p == Qnil) {
3107 *ary_p = rb_ary_new();
3108 }
3109
3110 if (DECORATOR_P(sname, dname)) {
3111 v = rb_str_new_cstr(dname);
3112 }
3113 else {
3114 v = rb_assoc_new(make_encobj(sname), make_encobj(dname));
3115 }
3116 rb_ary_store(*ary_p, depth, v);
3117 }
3118
3119 /*
3120 * call-seq:
3121 * Encoding::Converter.search_convpath(source_encoding, destination_encoding) -> ary
3122 * Encoding::Converter.search_convpath(source_encoding, destination_encoding, opt) -> ary
3123 *
3124 * Returns a conversion path.
3125 *
3126 * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP")
3127 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3128 * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>]]
3129 *
3130 * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", universal_newline: true)
3131 * or
3132 * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", newline: :universal)
3133 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3134 * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
3135 * # "universal_newline"]
3136 *
3137 * p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", universal_newline: true)
3138 * or
3139 * p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", newline: :universal)
3140 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3141 * # "universal_newline",
3142 * # [#<Encoding:UTF-8>, #<Encoding:UTF-32BE>]]
3143 */
3144 static VALUE
econv_s_search_convpath(int argc,VALUE * argv,VALUE klass)3145 econv_s_search_convpath(int argc, VALUE *argv, VALUE klass)
3146 {
3147 VALUE snamev, dnamev;
3148 const char *sname, *dname;
3149 rb_encoding *senc, *denc;
3150 int ecflags;
3151 VALUE ecopts;
3152 VALUE convpath;
3153
3154 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3155
3156 convpath = Qnil;
3157 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3158
3159 if (NIL_P(convpath)) {
3160 VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
3161 RB_GC_GUARD(snamev);
3162 RB_GC_GUARD(dnamev);
3163 rb_exc_raise(exc);
3164 }
3165
3166 if (decorate_convpath(convpath, ecflags) == -1) {
3167 VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
3168 RB_GC_GUARD(snamev);
3169 RB_GC_GUARD(dnamev);
3170 rb_exc_raise(exc);
3171 }
3172
3173 return convpath;
3174 }
3175
3176 /*
3177 * Check the existence of a conversion path.
3178 * Returns the number of converters in the conversion path.
3179 * result: >=0:success -1:failure
3180 */
3181 int
rb_econv_has_convpath_p(const char * from_encoding,const char * to_encoding)3182 rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding)
3183 {
3184 VALUE convpath = Qnil;
3185 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3186 &convpath);
3187 return RTEST(convpath);
3188 }
3189
3190 struct rb_econv_init_by_convpath_t {
3191 rb_econv_t *ec;
3192 int index;
3193 int ret;
3194 };
3195
3196 static void
rb_econv_init_by_convpath_i(const char * sname,const char * dname,int depth,void * arg)3197 rb_econv_init_by_convpath_i(const char *sname, const char *dname, int depth, void *arg)
3198 {
3199 struct rb_econv_init_by_convpath_t *a = (struct rb_econv_init_by_convpath_t *)arg;
3200 int ret;
3201
3202 if (a->ret == -1)
3203 return;
3204
3205 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3206
3207 a->ret = ret;
3208 return;
3209 }
3210
3211 static rb_econv_t *
rb_econv_init_by_convpath(VALUE self,VALUE convpath,const char ** sname_p,const char ** dname_p,rb_encoding ** senc_p,rb_encoding ** denc_p)3212 rb_econv_init_by_convpath(VALUE self, VALUE convpath,
3213 const char **sname_p, const char **dname_p,
3214 rb_encoding **senc_p, rb_encoding**denc_p)
3215 {
3216 rb_econv_t *ec;
3217 long i;
3218 int ret, first=1;
3219 VALUE elt;
3220 rb_encoding *senc = 0, *denc = 0;
3221 const char *sname, *dname;
3222
3223 ec = rb_econv_alloc(RARRAY_LENINT(convpath));
3224 DATA_PTR(self) = ec;
3225
3226 for (i = 0; i < RARRAY_LEN(convpath); i++) {
3227 VALUE snamev, dnamev;
3228 VALUE pair;
3229 elt = rb_ary_entry(convpath, i);
3230 if (!NIL_P(pair = rb_check_array_type(elt))) {
3231 if (RARRAY_LEN(pair) != 2)
3232 rb_raise(rb_eArgError, "not a 2-element array in convpath");
3233 snamev = rb_ary_entry(pair, 0);
3234 enc_arg(&snamev, &sname, &senc);
3235 dnamev = rb_ary_entry(pair, 1);
3236 enc_arg(&dnamev, &dname, &denc);
3237 }
3238 else {
3239 sname = "";
3240 dname = StringValueCStr(elt);
3241 }
3242 if (DECORATOR_P(sname, dname)) {
3243 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3244 if (ret == -1) {
3245 VALUE msg = rb_sprintf("decoration failed: %s", dname);
3246 RB_GC_GUARD(snamev);
3247 RB_GC_GUARD(dnamev);
3248 rb_exc_raise(rb_exc_new_str(rb_eArgError, msg));
3249 }
3250 }
3251 else {
3252 int j = ec->num_trans;
3253 struct rb_econv_init_by_convpath_t arg;
3254 arg.ec = ec;
3255 arg.index = ec->num_trans;
3256 arg.ret = 0;
3257 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3258 if (ret == -1 || arg.ret == -1) {
3259 VALUE msg = rb_sprintf("adding conversion failed: %s to %s", sname, dname);
3260 RB_GC_GUARD(snamev);
3261 RB_GC_GUARD(dnamev);
3262 rb_exc_raise(rb_exc_new_str(rb_eArgError, msg));
3263 }
3264 if (first) {
3265 first = 0;
3266 *senc_p = senc;
3267 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3268 }
3269 *denc_p = denc;
3270 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3271 }
3272 }
3273
3274 if (first) {
3275 *senc_p = NULL;
3276 *denc_p = NULL;
3277 *sname_p = "";
3278 *dname_p = "";
3279 }
3280
3281 ec->source_encoding_name = *sname_p;
3282 ec->destination_encoding_name = *dname_p;
3283
3284 return ec;
3285 }
3286
3287 /*
3288 * call-seq:
3289 * Encoding::Converter.new(source_encoding, destination_encoding)
3290 * Encoding::Converter.new(source_encoding, destination_encoding, opt)
3291 * Encoding::Converter.new(convpath)
3292 *
3293 * possible options elements:
3294 * hash form:
3295 * :invalid => nil # raise error on invalid byte sequence (default)
3296 * :invalid => :replace # replace invalid byte sequence
3297 * :undef => nil # raise error on undefined conversion (default)
3298 * :undef => :replace # replace undefined conversion
3299 * :replace => string # replacement string ("?" or "\uFFFD" if not specified)
3300 * :newline => :universal # decorator for converting CRLF and CR to LF
3301 * :newline => :crlf # decorator for converting LF to CRLF
3302 * :newline => :cr # decorator for converting LF to CR
3303 * :universal_newline => true # decorator for converting CRLF and CR to LF
3304 * :crlf_newline => true # decorator for converting LF to CRLF
3305 * :cr_newline => true # decorator for converting LF to CR
3306 * :xml => :text # escape as XML CharData.
3307 * :xml => :attr # escape as XML AttValue
3308 * integer form:
3309 * Encoding::Converter::INVALID_REPLACE
3310 * Encoding::Converter::UNDEF_REPLACE
3311 * Encoding::Converter::UNDEF_HEX_CHARREF
3312 * Encoding::Converter::UNIVERSAL_NEWLINE_DECORATOR
3313 * Encoding::Converter::CRLF_NEWLINE_DECORATOR
3314 * Encoding::Converter::CR_NEWLINE_DECORATOR
3315 * Encoding::Converter::XML_TEXT_DECORATOR
3316 * Encoding::Converter::XML_ATTR_CONTENT_DECORATOR
3317 * Encoding::Converter::XML_ATTR_QUOTE_DECORATOR
3318 *
3319 * Encoding::Converter.new creates an instance of Encoding::Converter.
3320 *
3321 * Source_encoding and destination_encoding should be a string or
3322 * Encoding object.
3323 *
3324 * opt should be nil, a hash or an integer.
3325 *
3326 * convpath should be an array.
3327 * convpath may contain
3328 * - two-element arrays which contain encodings or encoding names, or
3329 * - strings representing decorator names.
3330 *
3331 * Encoding::Converter.new optionally takes an option.
3332 * The option should be a hash or an integer.
3333 * The option hash can contain :invalid => nil, etc.
3334 * The option integer should be logical-or of constants such as
3335 * Encoding::Converter::INVALID_REPLACE, etc.
3336 *
3337 * [:invalid => nil]
3338 * Raise error on invalid byte sequence. This is a default behavior.
3339 * [:invalid => :replace]
3340 * Replace invalid byte sequence by replacement string.
3341 * [:undef => nil]
3342 * Raise an error if a character in source_encoding is not defined in destination_encoding.
3343 * This is a default behavior.
3344 * [:undef => :replace]
3345 * Replace undefined character in destination_encoding with replacement string.
3346 * [:replace => string]
3347 * Specify the replacement string.
3348 * If not specified, "\uFFFD" is used for Unicode encodings and "?" for others.
3349 * [:universal_newline => true]
3350 * Convert CRLF and CR to LF.
3351 * [:crlf_newline => true]
3352 * Convert LF to CRLF.
3353 * [:cr_newline => true]
3354 * Convert LF to CR.
3355 * [:xml => :text]
3356 * Escape as XML CharData.
3357 * This form can be used as an HTML 4.0 #PCDATA.
3358 * - '&' -> '&'
3359 * - '<' -> '<'
3360 * - '>' -> '>'
3361 * - undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;
3362 * [:xml => :attr]
3363 * Escape as XML AttValue.
3364 * The converted result is quoted as "...".
3365 * This form can be used as an HTML 4.0 attribute value.
3366 * - '&' -> '&'
3367 * - '<' -> '<'
3368 * - '>' -> '>'
3369 * - '"' -> '"'
3370 * - undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;
3371 *
3372 * Examples:
3373 * # UTF-16BE to UTF-8
3374 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
3375 *
3376 * # Usually, decorators such as newline conversion are inserted last.
3377 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8", :universal_newline => true)
3378 * p ec.convpath #=> [[#<Encoding:UTF-16BE>, #<Encoding:UTF-8>],
3379 * # "universal_newline"]
3380 *
3381 * # But, if the last encoding is ASCII incompatible,
3382 * # decorators are inserted before the last conversion.
3383 * ec = Encoding::Converter.new("UTF-8", "UTF-16BE", :crlf_newline => true)
3384 * p ec.convpath #=> ["crlf_newline",
3385 * # [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]
3386 *
3387 * # Conversion path can be specified directly.
3388 * ec = Encoding::Converter.new(["universal_newline", ["EUC-JP", "UTF-8"], ["UTF-8", "UTF-16BE"]])
3389 * p ec.convpath #=> ["universal_newline",
3390 * # [#<Encoding:EUC-JP>, #<Encoding:UTF-8>],
3391 * # [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]
3392 */
3393 static VALUE
econv_init(int argc,VALUE * argv,VALUE self)3394 econv_init(int argc, VALUE *argv, VALUE self)
3395 {
3396 VALUE ecopts;
3397 VALUE snamev, dnamev;
3398 const char *sname, *dname;
3399 rb_encoding *senc, *denc;
3400 rb_econv_t *ec;
3401 int ecflags;
3402 VALUE convpath;
3403
3404 if (rb_check_typeddata(self, &econv_data_type)) {
3405 rb_raise(rb_eTypeError, "already initialized");
3406 }
3407
3408 if (argc == 1 && !NIL_P(convpath = rb_check_array_type(argv[0]))) {
3409 ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
3410 ecflags = 0;
3411 ecopts = Qnil;
3412 }
3413 else {
3414 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3415 ec = rb_econv_open_opts(sname, dname, ecflags, ecopts);
3416 }
3417
3418 if (!ec) {
3419 VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
3420 RB_GC_GUARD(snamev);
3421 RB_GC_GUARD(dnamev);
3422 rb_exc_raise(exc);
3423 }
3424
3425 if (!DECORATOR_P(sname, dname)) {
3426 if (!senc)
3427 senc = make_dummy_encoding(sname);
3428 if (!denc)
3429 denc = make_dummy_encoding(dname);
3430 RB_GC_GUARD(snamev);
3431 RB_GC_GUARD(dnamev);
3432 }
3433
3434 ec->source_encoding = senc;
3435 ec->destination_encoding = denc;
3436
3437 DATA_PTR(self) = ec;
3438
3439 return self;
3440 }
3441
3442 /*
3443 * call-seq:
3444 * ec.inspect -> string
3445 *
3446 * Returns a printable version of <i>ec</i>
3447 *
3448 * ec = Encoding::Converter.new("iso-8859-1", "utf-8")
3449 * puts ec.inspect #=> #<Encoding::Converter: ISO-8859-1 to UTF-8>
3450 *
3451 */
3452 static VALUE
econv_inspect(VALUE self)3453 econv_inspect(VALUE self)
3454 {
3455 const char *cname = rb_obj_classname(self);
3456 rb_econv_t *ec;
3457
3458 TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
3459 if (!ec)
3460 return rb_sprintf("#<%s: uninitialized>", cname);
3461 else {
3462 const char *sname = ec->source_encoding_name;
3463 const char *dname = ec->destination_encoding_name;
3464 VALUE str;
3465 str = rb_sprintf("#<%s: ", cname);
3466 econv_description(sname, dname, ec->flags, str);
3467 rb_str_cat2(str, ">");
3468 return str;
3469 }
3470 }
3471
3472 static rb_econv_t *
check_econv(VALUE self)3473 check_econv(VALUE self)
3474 {
3475 rb_econv_t *ec;
3476
3477 TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
3478 if (!ec) {
3479 rb_raise(rb_eTypeError, "uninitialized encoding converter");
3480 }
3481 return ec;
3482 }
3483
3484 /*
3485 * call-seq:
3486 * ec.source_encoding -> encoding
3487 *
3488 * Returns the source encoding as an Encoding object.
3489 */
3490 static VALUE
econv_source_encoding(VALUE self)3491 econv_source_encoding(VALUE self)
3492 {
3493 rb_econv_t *ec = check_econv(self);
3494 if (!ec->source_encoding)
3495 return Qnil;
3496 return rb_enc_from_encoding(ec->source_encoding);
3497 }
3498
3499 /*
3500 * call-seq:
3501 * ec.destination_encoding -> encoding
3502 *
3503 * Returns the destination encoding as an Encoding object.
3504 */
3505 static VALUE
econv_destination_encoding(VALUE self)3506 econv_destination_encoding(VALUE self)
3507 {
3508 rb_econv_t *ec = check_econv(self);
3509 if (!ec->destination_encoding)
3510 return Qnil;
3511 return rb_enc_from_encoding(ec->destination_encoding);
3512 }
3513
3514 /*
3515 * call-seq:
3516 * ec.convpath -> ary
3517 *
3518 * Returns the conversion path of ec.
3519 *
3520 * The result is an array of conversions.
3521 *
3522 * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP", crlf_newline: true)
3523 * p ec.convpath
3524 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3525 * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
3526 * # "crlf_newline"]
3527 *
3528 * Each element of the array is a pair of encodings or a string.
3529 * A pair means an encoding conversion.
3530 * A string means a decorator.
3531 *
3532 * In the above example, [#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>] means
3533 * a converter from ISO-8859-1 to UTF-8.
3534 * "crlf_newline" means newline converter from LF to CRLF.
3535 */
3536 static VALUE
econv_convpath(VALUE self)3537 econv_convpath(VALUE self)
3538 {
3539 rb_econv_t *ec = check_econv(self);
3540 VALUE result;
3541 int i;
3542
3543 result = rb_ary_new();
3544 for (i = 0; i < ec->num_trans; i++) {
3545 const rb_transcoder *tr = ec->elems[i].tc->transcoder;
3546 VALUE v;
3547 if (DECORATOR_P(tr->src_encoding, tr->dst_encoding))
3548 v = rb_str_new_cstr(tr->dst_encoding);
3549 else
3550 v = rb_assoc_new(make_encobj(tr->src_encoding), make_encobj(tr->dst_encoding));
3551 rb_ary_push(result, v);
3552 }
3553 return result;
3554 }
3555
3556 /*
3557 * call-seq:
3558 * ec == other -> true or false
3559 */
3560 static VALUE
econv_equal(VALUE self,VALUE other)3561 econv_equal(VALUE self, VALUE other)
3562 {
3563 rb_econv_t *ec1 = check_econv(self);
3564 rb_econv_t *ec2;
3565 int i;
3566
3567 if (!rb_typeddata_is_kind_of(other, &econv_data_type)) {
3568 return Qnil;
3569 }
3570 ec2 = DATA_PTR(other);
3571 if (!ec2) return Qfalse;
3572 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3573 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3574 return Qfalse;
3575 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3576 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3577 return Qfalse;
3578 if (ec1->flags != ec2->flags) return Qfalse;
3579 if (ec1->replacement_enc != ec2->replacement_enc &&
3580 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3581 return Qfalse;
3582 if (ec1->replacement_len != ec2->replacement_len) return Qfalse;
3583 if (ec1->replacement_str != ec2->replacement_str &&
3584 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3585 return Qfalse;
3586
3587 if (ec1->num_trans != ec2->num_trans) return Qfalse;
3588 for (i = 0; i < ec1->num_trans; i++) {
3589 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3590 return Qfalse;
3591 }
3592 return Qtrue;
3593 }
3594
3595 static VALUE
econv_result_to_symbol(rb_econv_result_t res)3596 econv_result_to_symbol(rb_econv_result_t res)
3597 {
3598 switch (res) {
3599 case econv_invalid_byte_sequence: return sym_invalid_byte_sequence;
3600 case econv_incomplete_input: return sym_incomplete_input;
3601 case econv_undefined_conversion: return sym_undefined_conversion;
3602 case econv_destination_buffer_full: return sym_destination_buffer_full;
3603 case econv_source_buffer_empty: return sym_source_buffer_empty;
3604 case econv_finished: return sym_finished;
3605 case econv_after_output: return sym_after_output;
3606 default: return INT2NUM(res); /* should not be reached */
3607 }
3608 }
3609
3610 /*
3611 * call-seq:
3612 * ec.primitive_convert(source_buffer, destination_buffer) -> symbol
3613 * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset) -> symbol
3614 * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize) -> symbol
3615 * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize, opt) -> symbol
3616 *
3617 * possible opt elements:
3618 * hash form:
3619 * :partial_input => true # source buffer may be part of larger source
3620 * :after_output => true # stop conversion after output before input
3621 * integer form:
3622 * Encoding::Converter::PARTIAL_INPUT
3623 * Encoding::Converter::AFTER_OUTPUT
3624 *
3625 * possible results:
3626 * :invalid_byte_sequence
3627 * :incomplete_input
3628 * :undefined_conversion
3629 * :after_output
3630 * :destination_buffer_full
3631 * :source_buffer_empty
3632 * :finished
3633 *
3634 * primitive_convert converts source_buffer into destination_buffer.
3635 *
3636 * source_buffer should be a string or nil.
3637 * nil means an empty string.
3638 *
3639 * destination_buffer should be a string.
3640 *
3641 * destination_byteoffset should be an integer or nil.
3642 * nil means the end of destination_buffer.
3643 * If it is omitted, nil is assumed.
3644 *
3645 * destination_bytesize should be an integer or nil.
3646 * nil means unlimited.
3647 * If it is omitted, nil is assumed.
3648 *
3649 * opt should be nil, a hash or an integer.
3650 * nil means no flags.
3651 * If it is omitted, nil is assumed.
3652 *
3653 * primitive_convert converts the content of source_buffer from beginning
3654 * and store the result into destination_buffer.
3655 *
3656 * destination_byteoffset and destination_bytesize specify the region which
3657 * the converted result is stored.
3658 * destination_byteoffset specifies the start position in destination_buffer in bytes.
3659 * If destination_byteoffset is nil,
3660 * destination_buffer.bytesize is used for appending the result.
3661 * destination_bytesize specifies maximum number of bytes.
3662 * If destination_bytesize is nil,
3663 * destination size is unlimited.
3664 * After conversion, destination_buffer is resized to
3665 * destination_byteoffset + actually produced number of bytes.
3666 * Also destination_buffer's encoding is set to destination_encoding.
3667 *
3668 * primitive_convert drops the converted part of source_buffer.
3669 * the dropped part is converted in destination_buffer or
3670 * buffered in Encoding::Converter object.
3671 *
3672 * primitive_convert stops conversion when one of following condition met.
3673 * - invalid byte sequence found in source buffer (:invalid_byte_sequence)
3674 * +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
3675 * - unexpected end of source buffer (:incomplete_input)
3676 * this occur only when :partial_input is not specified.
3677 * +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
3678 * - character not representable in output encoding (:undefined_conversion)
3679 * +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
3680 * - after some output is generated, before input is done (:after_output)
3681 * this occur only when :after_output is specified.
3682 * - destination buffer is full (:destination_buffer_full)
3683 * this occur only when destination_bytesize is non-nil.
3684 * - source buffer is empty (:source_buffer_empty)
3685 * this occur only when :partial_input is specified.
3686 * - conversion is finished (:finished)
3687 *
3688 * example:
3689 * ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
3690 * ret = ec.primitive_convert(src="pi", dst="", nil, 100)
3691 * p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"]
3692 *
3693 * ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
3694 * ret = ec.primitive_convert(src="pi", dst="", nil, 1)
3695 * p [ret, src, dst] #=> [:destination_buffer_full, "i", "\x00"]
3696 * ret = ec.primitive_convert(src, dst="", nil, 1)
3697 * p [ret, src, dst] #=> [:destination_buffer_full, "", "p"]
3698 * ret = ec.primitive_convert(src, dst="", nil, 1)
3699 * p [ret, src, dst] #=> [:destination_buffer_full, "", "\x00"]
3700 * ret = ec.primitive_convert(src, dst="", nil, 1)
3701 * p [ret, src, dst] #=> [:finished, "", "i"]
3702 *
3703 */
3704 static VALUE
econv_primitive_convert(int argc,VALUE * argv,VALUE self)3705 econv_primitive_convert(int argc, VALUE *argv, VALUE self)
3706 {
3707 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3708 rb_econv_t *ec = check_econv(self);
3709 rb_econv_result_t res;
3710 const unsigned char *ip, *is;
3711 unsigned char *op, *os;
3712 long output_byteoffset, output_bytesize;
3713 unsigned long output_byteend;
3714 int flags;
3715
3716 argc = rb_scan_args(argc, argv, "23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3717
3718 if (NIL_P(output_byteoffset_v))
3719 output_byteoffset = 0; /* dummy */
3720 else
3721 output_byteoffset = NUM2LONG(output_byteoffset_v);
3722
3723 if (NIL_P(output_bytesize_v))
3724 output_bytesize = 0; /* dummy */
3725 else
3726 output_bytesize = NUM2LONG(output_bytesize_v);
3727
3728 if (!NIL_P(flags_v)) {
3729 if (!NIL_P(opt)) {
3730 rb_error_arity(argc + 1, 2, 5);
3731 }
3732 flags = NUM2INT(rb_to_int(flags_v));
3733 }
3734 else if (!NIL_P(opt)) {
3735 VALUE v;
3736 flags = 0;
3737 v = rb_hash_aref(opt, sym_partial_input);
3738 if (RTEST(v))
3739 flags |= ECONV_PARTIAL_INPUT;
3740 v = rb_hash_aref(opt, sym_after_output);
3741 if (RTEST(v))
3742 flags |= ECONV_AFTER_OUTPUT;
3743 }
3744 else {
3745 flags = 0;
3746 }
3747
3748 StringValue(output);
3749 if (!NIL_P(input))
3750 StringValue(input);
3751 rb_str_modify(output);
3752
3753 if (NIL_P(output_bytesize_v)) {
3754 output_bytesize = RSTRING_EMBED_LEN_MAX;
3755 if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input))
3756 output_bytesize = RSTRING_LEN(input);
3757 }
3758
3759 retry:
3760
3761 if (NIL_P(output_byteoffset_v))
3762 output_byteoffset = RSTRING_LEN(output);
3763
3764 if (output_byteoffset < 0)
3765 rb_raise(rb_eArgError, "negative output_byteoffset");
3766
3767 if (RSTRING_LEN(output) < output_byteoffset)
3768 rb_raise(rb_eArgError, "output_byteoffset too big");
3769
3770 if (output_bytesize < 0)
3771 rb_raise(rb_eArgError, "negative output_bytesize");
3772
3773 output_byteend = (unsigned long)output_byteoffset +
3774 (unsigned long)output_bytesize;
3775
3776 if (output_byteend < (unsigned long)output_byteoffset ||
3777 LONG_MAX < output_byteend)
3778 rb_raise(rb_eArgError, "output_byteoffset+output_bytesize too big");
3779
3780 if (rb_str_capacity(output) < output_byteend)
3781 rb_str_resize(output, output_byteend);
3782
3783 if (NIL_P(input)) {
3784 ip = is = NULL;
3785 }
3786 else {
3787 ip = (const unsigned char *)RSTRING_PTR(input);
3788 is = ip + RSTRING_LEN(input);
3789 }
3790
3791 op = (unsigned char *)RSTRING_PTR(output) + output_byteoffset;
3792 os = op + output_bytesize;
3793
3794 res = rb_econv_convert(ec, &ip, is, &op, os, flags);
3795 rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output));
3796 if (!NIL_P(input)) {
3797 OBJ_INFECT_RAW(output, input);
3798 rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input));
3799 }
3800
3801 if (NIL_P(output_bytesize_v) && res == econv_destination_buffer_full) {
3802 if (LONG_MAX / 2 < output_bytesize)
3803 rb_raise(rb_eArgError, "too long conversion result");
3804 output_bytesize *= 2;
3805 output_byteoffset_v = Qnil;
3806 goto retry;
3807 }
3808
3809 if (ec->destination_encoding) {
3810 rb_enc_associate(output, ec->destination_encoding);
3811 }
3812
3813 return econv_result_to_symbol(res);
3814 }
3815
3816 /*
3817 * call-seq:
3818 * ec.convert(source_string) -> destination_string
3819 *
3820 * Convert source_string and return destination_string.
3821 *
3822 * source_string is assumed as a part of source.
3823 * i.e. :partial_input=>true is specified internally.
3824 * finish method should be used last.
3825 *
3826 * ec = Encoding::Converter.new("utf-8", "euc-jp")
3827 * puts ec.convert("\u3042").dump #=> "\xA4\xA2"
3828 * puts ec.finish.dump #=> ""
3829 *
3830 * ec = Encoding::Converter.new("euc-jp", "utf-8")
3831 * puts ec.convert("\xA4").dump #=> ""
3832 * puts ec.convert("\xA2").dump #=> "\xE3\x81\x82"
3833 * puts ec.finish.dump #=> ""
3834 *
3835 * ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
3836 * puts ec.convert("\xE3").dump #=> "".force_encoding("ISO-2022-JP")
3837 * puts ec.convert("\x81").dump #=> "".force_encoding("ISO-2022-JP")
3838 * puts ec.convert("\x82").dump #=> "\e$B$\"".force_encoding("ISO-2022-JP")
3839 * puts ec.finish.dump #=> "\e(B".force_encoding("ISO-2022-JP")
3840 *
3841 * If a conversion error occur,
3842 * Encoding::UndefinedConversionError or
3843 * Encoding::InvalidByteSequenceError is raised.
3844 * Encoding::Converter#convert doesn't supply methods to recover or restart
3845 * from these exceptions.
3846 * When you want to handle these conversion errors,
3847 * use Encoding::Converter#primitive_convert.
3848 *
3849 */
3850 static VALUE
econv_convert(VALUE self,VALUE source_string)3851 econv_convert(VALUE self, VALUE source_string)
3852 {
3853 VALUE ret, dst;
3854 VALUE av[5];
3855 int ac;
3856 rb_econv_t *ec = check_econv(self);
3857
3858 StringValue(source_string);
3859
3860 dst = rb_str_new(NULL, 0);
3861
3862 av[0] = rb_str_dup(source_string);
3863 av[1] = dst;
3864 av[2] = Qnil;
3865 av[3] = Qnil;
3866 av[4] = INT2NUM(ECONV_PARTIAL_INPUT);
3867 ac = 5;
3868
3869 ret = econv_primitive_convert(ac, av, self);
3870
3871 if (ret == sym_invalid_byte_sequence ||
3872 ret == sym_undefined_conversion ||
3873 ret == sym_incomplete_input) {
3874 VALUE exc = make_econv_exception(ec);
3875 rb_exc_raise(exc);
3876 }
3877
3878 if (ret == sym_finished) {
3879 rb_raise(rb_eArgError, "converter already finished");
3880 }
3881
3882 if (ret != sym_source_buffer_empty) {
3883 rb_bug("unexpected result of econv_primitive_convert");
3884 }
3885
3886 return dst;
3887 }
3888
3889 /*
3890 * call-seq:
3891 * ec.finish -> string
3892 *
3893 * Finishes the converter.
3894 * It returns the last part of the converted string.
3895 *
3896 * ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
3897 * p ec.convert("\u3042") #=> "\e$B$\""
3898 * p ec.finish #=> "\e(B"
3899 */
3900 static VALUE
econv_finish(VALUE self)3901 econv_finish(VALUE self)
3902 {
3903 VALUE ret, dst;
3904 VALUE av[5];
3905 int ac;
3906 rb_econv_t *ec = check_econv(self);
3907
3908 dst = rb_str_new(NULL, 0);
3909
3910 av[0] = Qnil;
3911 av[1] = dst;
3912 av[2] = Qnil;
3913 av[3] = Qnil;
3914 av[4] = INT2FIX(0);
3915 ac = 5;
3916
3917 ret = econv_primitive_convert(ac, av, self);
3918
3919 if (ret == sym_invalid_byte_sequence ||
3920 ret == sym_undefined_conversion ||
3921 ret == sym_incomplete_input) {
3922 VALUE exc = make_econv_exception(ec);
3923 rb_exc_raise(exc);
3924 }
3925
3926 if (ret != sym_finished) {
3927 rb_bug("unexpected result of econv_primitive_convert");
3928 }
3929
3930 return dst;
3931 }
3932
3933 /*
3934 * call-seq:
3935 * ec.primitive_errinfo -> array
3936 *
3937 * primitive_errinfo returns important information regarding the last error
3938 * as a 5-element array:
3939 *
3940 * [result, enc1, enc2, error_bytes, readagain_bytes]
3941 *
3942 * result is the last result of primitive_convert.
3943 *
3944 * Other elements are only meaningful when result is
3945 * :invalid_byte_sequence, :incomplete_input or :undefined_conversion.
3946 *
3947 * enc1 and enc2 indicate a conversion step as a pair of strings.
3948 * For example, a converter from EUC-JP to ISO-8859-1 converts
3949 * a string as follows: EUC-JP -> UTF-8 -> ISO-8859-1.
3950 * So [enc1, enc2] is either ["EUC-JP", "UTF-8"] or ["UTF-8", "ISO-8859-1"].
3951 *
3952 * error_bytes and readagain_bytes indicate the byte sequences which caused the error.
3953 * error_bytes is discarded portion.
3954 * readagain_bytes is buffered portion which is read again on next conversion.
3955 *
3956 * Example:
3957 *
3958 * # \xff is invalid as EUC-JP.
3959 * ec = Encoding::Converter.new("EUC-JP", "Shift_JIS")
3960 * ec.primitive_convert(src="\xff", dst="", nil, 10)
3961 * p ec.primitive_errinfo
3962 * #=> [:invalid_byte_sequence, "EUC-JP", "UTF-8", "\xFF", ""]
3963 *
3964 * # HIRAGANA LETTER A (\xa4\xa2 in EUC-JP) is not representable in ISO-8859-1.
3965 * # Since this error is occur in UTF-8 to ISO-8859-1 conversion,
3966 * # error_bytes is HIRAGANA LETTER A in UTF-8 (\xE3\x81\x82).
3967 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
3968 * ec.primitive_convert(src="\xa4\xa2", dst="", nil, 10)
3969 * p ec.primitive_errinfo
3970 * #=> [:undefined_conversion, "UTF-8", "ISO-8859-1", "\xE3\x81\x82", ""]
3971 *
3972 * # partial character is invalid
3973 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
3974 * ec.primitive_convert(src="\xa4", dst="", nil, 10)
3975 * p ec.primitive_errinfo
3976 * #=> [:incomplete_input, "EUC-JP", "UTF-8", "\xA4", ""]
3977 *
3978 * # Encoding::Converter::PARTIAL_INPUT prevents invalid errors by
3979 * # partial characters.
3980 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
3981 * ec.primitive_convert(src="\xa4", dst="", nil, 10, Encoding::Converter::PARTIAL_INPUT)
3982 * p ec.primitive_errinfo
3983 * #=> [:source_buffer_empty, nil, nil, nil, nil]
3984 *
3985 * # \xd8\x00\x00@ is invalid as UTF-16BE because
3986 * # no low surrogate after high surrogate (\xd8\x00).
3987 * # It is detected by 3rd byte (\00) which is part of next character.
3988 * # So the high surrogate (\xd8\x00) is discarded and
3989 * # the 3rd byte is read again later.
3990 * # Since the byte is buffered in ec, it is dropped from src.
3991 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
3992 * ec.primitive_convert(src="\xd8\x00\x00@", dst="", nil, 10)
3993 * p ec.primitive_errinfo
3994 * #=> [:invalid_byte_sequence, "UTF-16BE", "UTF-8", "\xD8\x00", "\x00"]
3995 * p src
3996 * #=> "@"
3997 *
3998 * # Similar to UTF-16BE, \x00\xd8@\x00 is invalid as UTF-16LE.
3999 * # The problem is detected by 4th byte.
4000 * ec = Encoding::Converter.new("UTF-16LE", "UTF-8")
4001 * ec.primitive_convert(src="\x00\xd8@\x00", dst="", nil, 10)
4002 * p ec.primitive_errinfo
4003 * #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "@\x00"]
4004 * p src
4005 * #=> ""
4006 *
4007 */
4008 static VALUE
econv_primitive_errinfo(VALUE self)4009 econv_primitive_errinfo(VALUE self)
4010 {
4011 rb_econv_t *ec = check_econv(self);
4012
4013 VALUE ary;
4014
4015 ary = rb_ary_new2(5);
4016
4017 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4018 rb_ary_store(ary, 4, Qnil);
4019
4020 if (ec->last_error.source_encoding)
4021 rb_ary_store(ary, 1, rb_str_new2(ec->last_error.source_encoding));
4022
4023 if (ec->last_error.destination_encoding)
4024 rb_ary_store(ary, 2, rb_str_new2(ec->last_error.destination_encoding));
4025
4026 if (ec->last_error.error_bytes_start) {
4027 rb_ary_store(ary, 3, rb_str_new((const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4028 rb_ary_store(ary, 4, rb_str_new((const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4029 }
4030
4031 return ary;
4032 }
4033
4034 /*
4035 * call-seq:
4036 * ec.insert_output(string) -> nil
4037 *
4038 * Inserts string into the encoding converter.
4039 * The string will be converted to the destination encoding and
4040 * output on later conversions.
4041 *
4042 * If the destination encoding is stateful,
4043 * string is converted according to the state and the state is updated.
4044 *
4045 * This method should be used only when a conversion error occurs.
4046 *
4047 * ec = Encoding::Converter.new("utf-8", "iso-8859-1")
4048 * src = "HIRAGANA LETTER A is \u{3042}."
4049 * dst = ""
4050 * p ec.primitive_convert(src, dst) #=> :undefined_conversion
4051 * puts "[#{dst.dump}, #{src.dump}]" #=> ["HIRAGANA LETTER A is ", "."]
4052 * ec.insert_output("<err>")
4053 * p ec.primitive_convert(src, dst) #=> :finished
4054 * puts "[#{dst.dump}, #{src.dump}]" #=> ["HIRAGANA LETTER A is <err>.", ""]
4055 *
4056 * ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
4057 * src = "\u{306F 3041 3068 2661 3002}" # U+2661 is not representable in iso-2022-jp
4058 * dst = ""
4059 * p ec.primitive_convert(src, dst) #=> :undefined_conversion
4060 * puts "[#{dst.dump}, #{src.dump}]" #=> ["\e$B$O$!$H".force_encoding("ISO-2022-JP"), "\xE3\x80\x82"]
4061 * ec.insert_output "?" # state change required to output "?".
4062 * p ec.primitive_convert(src, dst) #=> :finished
4063 * puts "[#{dst.dump}, #{src.dump}]" #=> ["\e$B$O$!$H\e(B?\e$B!#\e(B".force_encoding("ISO-2022-JP"), ""]
4064 *
4065 */
4066 static VALUE
econv_insert_output(VALUE self,VALUE string)4067 econv_insert_output(VALUE self, VALUE string)
4068 {
4069 const char *insert_enc;
4070
4071 int ret;
4072
4073 rb_econv_t *ec = check_econv(self);
4074
4075 StringValue(string);
4076 insert_enc = rb_econv_encoding_to_insert_output(ec);
4077 string = rb_str_encode(string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0, Qnil);
4078
4079 ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc);
4080 if (ret == -1) {
4081 rb_raise(rb_eArgError, "too big string");
4082 }
4083
4084 return Qnil;
4085 }
4086
4087 /*
4088 * call-seq:
4089 * ec.putback -> string
4090 * ec.putback(max_numbytes) -> string
4091 *
4092 * Put back the bytes which will be converted.
4093 *
4094 * The bytes are caused by invalid_byte_sequence error.
4095 * When invalid_byte_sequence error, some bytes are discarded and
4096 * some bytes are buffered to be converted later.
4097 * The latter bytes can be put back.
4098 * It can be observed by
4099 * Encoding::InvalidByteSequenceError#readagain_bytes and
4100 * Encoding::Converter#primitive_errinfo.
4101 *
4102 * ec = Encoding::Converter.new("utf-16le", "iso-8859-1")
4103 * src = "\x00\xd8\x61\x00"
4104 * dst = ""
4105 * p ec.primitive_convert(src, dst) #=> :invalid_byte_sequence
4106 * p ec.primitive_errinfo #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "a\x00"]
4107 * p ec.putback #=> "a\x00"
4108 * p ec.putback #=> "" # no more bytes to put back
4109 *
4110 */
4111 static VALUE
econv_putback(int argc,VALUE * argv,VALUE self)4112 econv_putback(int argc, VALUE *argv, VALUE self)
4113 {
4114 rb_econv_t *ec = check_econv(self);
4115 int n;
4116 int putbackable;
4117 VALUE str, max;
4118
4119 if (!rb_check_arity(argc, 0, 1) || NIL_P(max = argv[0])) {
4120 n = rb_econv_putbackable(ec);
4121 }
4122 else {
4123 n = NUM2INT(max);
4124 putbackable = rb_econv_putbackable(ec);
4125 if (putbackable < n)
4126 n = putbackable;
4127 }
4128
4129 str = rb_str_new(NULL, n);
4130 rb_econv_putback(ec, (unsigned char *)RSTRING_PTR(str), n);
4131
4132 if (ec->source_encoding) {
4133 rb_enc_associate(str, ec->source_encoding);
4134 }
4135
4136 return str;
4137 }
4138
4139 /*
4140 * call-seq:
4141 * ec.last_error -> exception or nil
4142 *
4143 * Returns an exception object for the last conversion.
4144 * Returns nil if the last conversion did not produce an error.
4145 *
4146 * "error" means that
4147 * Encoding::InvalidByteSequenceError and Encoding::UndefinedConversionError for
4148 * Encoding::Converter#convert and
4149 * :invalid_byte_sequence, :incomplete_input and :undefined_conversion for
4150 * Encoding::Converter#primitive_convert.
4151 *
4152 * ec = Encoding::Converter.new("utf-8", "iso-8859-1")
4153 * p ec.primitive_convert(src="\xf1abcd", dst="") #=> :invalid_byte_sequence
4154 * p ec.last_error #=> #<Encoding::InvalidByteSequenceError: "\xF1" followed by "a" on UTF-8>
4155 * p ec.primitive_convert(src, dst, nil, 1) #=> :destination_buffer_full
4156 * p ec.last_error #=> nil
4157 *
4158 */
4159 static VALUE
econv_last_error(VALUE self)4160 econv_last_error(VALUE self)
4161 {
4162 rb_econv_t *ec = check_econv(self);
4163 VALUE exc;
4164
4165 exc = make_econv_exception(ec);
4166 if (NIL_P(exc))
4167 return Qnil;
4168 return exc;
4169 }
4170
4171 /*
4172 * call-seq:
4173 * ec.replacement -> string
4174 *
4175 * Returns the replacement string.
4176 *
4177 * ec = Encoding::Converter.new("euc-jp", "us-ascii")
4178 * p ec.replacement #=> "?"
4179 *
4180 * ec = Encoding::Converter.new("euc-jp", "utf-8")
4181 * p ec.replacement #=> "\uFFFD"
4182 */
4183 static VALUE
econv_get_replacement(VALUE self)4184 econv_get_replacement(VALUE self)
4185 {
4186 rb_econv_t *ec = check_econv(self);
4187 int ret;
4188 rb_encoding *enc;
4189
4190 ret = make_replacement(ec);
4191 if (ret == -1) {
4192 rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
4193 }
4194
4195 enc = rb_enc_find(ec->replacement_enc);
4196 return rb_enc_str_new((const char *)ec->replacement_str, (long)ec->replacement_len, enc);
4197 }
4198
4199 /*
4200 * call-seq:
4201 * ec.replacement = string
4202 *
4203 * Sets the replacement string.
4204 *
4205 * ec = Encoding::Converter.new("utf-8", "us-ascii", :undef => :replace)
4206 * ec.replacement = "<undef>"
4207 * p ec.convert("a \u3042 b") #=> "a <undef> b"
4208 */
4209 static VALUE
econv_set_replacement(VALUE self,VALUE arg)4210 econv_set_replacement(VALUE self, VALUE arg)
4211 {
4212 rb_econv_t *ec = check_econv(self);
4213 VALUE string = arg;
4214 int ret;
4215 rb_encoding *enc;
4216
4217 StringValue(string);
4218 enc = rb_enc_get(string);
4219
4220 ret = rb_econv_set_replacement(ec,
4221 (const unsigned char *)RSTRING_PTR(string),
4222 RSTRING_LEN(string),
4223 rb_enc_name(enc));
4224
4225 if (ret == -1) {
4226 /* xxx: rb_eInvalidByteSequenceError? */
4227 rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
4228 }
4229
4230 return arg;
4231 }
4232
4233 VALUE
rb_econv_make_exception(rb_econv_t * ec)4234 rb_econv_make_exception(rb_econv_t *ec)
4235 {
4236 return make_econv_exception(ec);
4237 }
4238
4239 void
rb_econv_check_error(rb_econv_t * ec)4240 rb_econv_check_error(rb_econv_t *ec)
4241 {
4242 VALUE exc;
4243
4244 exc = make_econv_exception(ec);
4245 if (NIL_P(exc))
4246 return;
4247 rb_exc_raise(exc);
4248 }
4249
4250 /*
4251 * call-seq:
4252 * ecerr.source_encoding_name -> string
4253 *
4254 * Returns the source encoding name as a string.
4255 */
4256 static VALUE
ecerr_source_encoding_name(VALUE self)4257 ecerr_source_encoding_name(VALUE self)
4258 {
4259 return rb_attr_get(self, rb_intern("source_encoding_name"));
4260 }
4261
4262 /*
4263 * call-seq:
4264 * ecerr.source_encoding -> encoding
4265 *
4266 * Returns the source encoding as an encoding object.
4267 *
4268 * Note that the result may not be equal to the source encoding of
4269 * the encoding converter if the conversion has multiple steps.
4270 *
4271 * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP") # ISO-8859-1 -> UTF-8 -> EUC-JP
4272 * begin
4273 * ec.convert("\xa0") # NO-BREAK SPACE, which is available in UTF-8 but not in EUC-JP.
4274 * rescue Encoding::UndefinedConversionError
4275 * p $!.source_encoding #=> #<Encoding:UTF-8>
4276 * p $!.destination_encoding #=> #<Encoding:EUC-JP>
4277 * p $!.source_encoding_name #=> "UTF-8"
4278 * p $!.destination_encoding_name #=> "EUC-JP"
4279 * end
4280 *
4281 */
4282 static VALUE
ecerr_source_encoding(VALUE self)4283 ecerr_source_encoding(VALUE self)
4284 {
4285 return rb_attr_get(self, rb_intern("source_encoding"));
4286 }
4287
4288 /*
4289 * call-seq:
4290 * ecerr.destination_encoding_name -> string
4291 *
4292 * Returns the destination encoding name as a string.
4293 */
4294 static VALUE
ecerr_destination_encoding_name(VALUE self)4295 ecerr_destination_encoding_name(VALUE self)
4296 {
4297 return rb_attr_get(self, rb_intern("destination_encoding_name"));
4298 }
4299
4300 /*
4301 * call-seq:
4302 * ecerr.destination_encoding -> string
4303 *
4304 * Returns the destination encoding as an encoding object.
4305 */
4306 static VALUE
ecerr_destination_encoding(VALUE self)4307 ecerr_destination_encoding(VALUE self)
4308 {
4309 return rb_attr_get(self, rb_intern("destination_encoding"));
4310 }
4311
4312 /*
4313 * call-seq:
4314 * ecerr.error_char -> string
4315 *
4316 * Returns the one-character string which cause Encoding::UndefinedConversionError.
4317 *
4318 * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP")
4319 * begin
4320 * ec.convert("\xa0")
4321 * rescue Encoding::UndefinedConversionError
4322 * puts $!.error_char.dump #=> "\xC2\xA0"
4323 * p $!.error_char.encoding #=> #<Encoding:UTF-8>
4324 * end
4325 *
4326 */
4327 static VALUE
ecerr_error_char(VALUE self)4328 ecerr_error_char(VALUE self)
4329 {
4330 return rb_attr_get(self, rb_intern("error_char"));
4331 }
4332
4333 /*
4334 * call-seq:
4335 * ecerr.error_bytes -> string
4336 *
4337 * Returns the discarded bytes when Encoding::InvalidByteSequenceError occurs.
4338 *
4339 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4340 * begin
4341 * ec.convert("abc\xA1\xFFdef")
4342 * rescue Encoding::InvalidByteSequenceError
4343 * p $! #=> #<Encoding::InvalidByteSequenceError: "\xA1" followed by "\xFF" on EUC-JP>
4344 * puts $!.error_bytes.dump #=> "\xA1"
4345 * puts $!.readagain_bytes.dump #=> "\xFF"
4346 * end
4347 */
4348 static VALUE
ecerr_error_bytes(VALUE self)4349 ecerr_error_bytes(VALUE self)
4350 {
4351 return rb_attr_get(self, rb_intern("error_bytes"));
4352 }
4353
4354 /*
4355 * call-seq:
4356 * ecerr.readagain_bytes -> string
4357 *
4358 * Returns the bytes to be read again when Encoding::InvalidByteSequenceError occurs.
4359 */
4360 static VALUE
ecerr_readagain_bytes(VALUE self)4361 ecerr_readagain_bytes(VALUE self)
4362 {
4363 return rb_attr_get(self, rb_intern("readagain_bytes"));
4364 }
4365
4366 /*
4367 * call-seq:
4368 * ecerr.incomplete_input? -> true or false
4369 *
4370 * Returns true if the invalid byte sequence error is caused by
4371 * premature end of string.
4372 *
4373 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4374 *
4375 * begin
4376 * ec.convert("abc\xA1z")
4377 * rescue Encoding::InvalidByteSequenceError
4378 * p $! #=> #<Encoding::InvalidByteSequenceError: "\xA1" followed by "z" on EUC-JP>
4379 * p $!.incomplete_input? #=> false
4380 * end
4381 *
4382 * begin
4383 * ec.convert("abc\xA1")
4384 * ec.finish
4385 * rescue Encoding::InvalidByteSequenceError
4386 * p $! #=> #<Encoding::InvalidByteSequenceError: incomplete "\xA1" on EUC-JP>
4387 * p $!.incomplete_input? #=> true
4388 * end
4389 */
4390 static VALUE
ecerr_incomplete_input(VALUE self)4391 ecerr_incomplete_input(VALUE self)
4392 {
4393 return rb_attr_get(self, rb_intern("incomplete_input"));
4394 }
4395
4396 /*
4397 * Document-class: Encoding::UndefinedConversionError
4398 *
4399 * Raised by Encoding and String methods when a transcoding operation
4400 * fails.
4401 */
4402
4403 /*
4404 * Document-class: Encoding::InvalidByteSequenceError
4405 *
4406 * Raised by Encoding and String methods when the string being
4407 * transcoded contains a byte invalid for the either the source or
4408 * target encoding.
4409 */
4410
4411 /*
4412 * Document-class: Encoding::ConverterNotFoundError
4413 *
4414 * Raised by transcoding methods when a named encoding does not
4415 * correspond with a known converter.
4416 */
4417
4418 #undef rb_intern
4419 void
Init_transcode(void)4420 Init_transcode(void)
4421 {
4422 transcoder_table = st_init_strcasetable();
4423
4424 sym_invalid = ID2SYM(rb_intern("invalid"));
4425 sym_undef = ID2SYM(rb_intern("undef"));
4426 sym_replace = ID2SYM(rb_intern("replace"));
4427 sym_fallback = ID2SYM(rb_intern("fallback"));
4428 sym_xml = ID2SYM(rb_intern("xml"));
4429 sym_text = ID2SYM(rb_intern("text"));
4430 sym_attr = ID2SYM(rb_intern("attr"));
4431
4432 sym_invalid_byte_sequence = ID2SYM(rb_intern("invalid_byte_sequence"));
4433 sym_undefined_conversion = ID2SYM(rb_intern("undefined_conversion"));
4434 sym_destination_buffer_full = ID2SYM(rb_intern("destination_buffer_full"));
4435 sym_source_buffer_empty = ID2SYM(rb_intern("source_buffer_empty"));
4436 sym_finished = ID2SYM(rb_intern("finished"));
4437 sym_after_output = ID2SYM(rb_intern("after_output"));
4438 sym_incomplete_input = ID2SYM(rb_intern("incomplete_input"));
4439 sym_universal_newline = ID2SYM(rb_intern("universal_newline"));
4440 sym_crlf_newline = ID2SYM(rb_intern("crlf_newline"));
4441 sym_cr_newline = ID2SYM(rb_intern("cr_newline"));
4442 sym_partial_input = ID2SYM(rb_intern("partial_input"));
4443
4444 #ifdef ENABLE_ECONV_NEWLINE_OPTION
4445 sym_newline = ID2SYM(rb_intern("newline"));
4446 sym_universal = ID2SYM(rb_intern("universal"));
4447 sym_crlf = ID2SYM(rb_intern("crlf"));
4448 sym_cr = ID2SYM(rb_intern("cr"));
4449 sym_lf = ID2SYM(rb_intern("lf"));
4450 #endif
4451
4452 InitVM(transcode);
4453 }
4454
4455 void
InitVM_transcode(void)4456 InitVM_transcode(void)
4457 {
4458 rb_eUndefinedConversionError = rb_define_class_under(rb_cEncoding, "UndefinedConversionError", rb_eEncodingError);
4459 rb_eInvalidByteSequenceError = rb_define_class_under(rb_cEncoding, "InvalidByteSequenceError", rb_eEncodingError);
4460 rb_eConverterNotFoundError = rb_define_class_under(rb_cEncoding, "ConverterNotFoundError", rb_eEncodingError);
4461
4462 rb_define_method(rb_cString, "encode", str_encode, -1);
4463 rb_define_method(rb_cString, "encode!", str_encode_bang, -1);
4464
4465 rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cData);
4466 rb_define_alloc_func(rb_cEncodingConverter, econv_s_allocate);
4467 rb_define_singleton_method(rb_cEncodingConverter, "asciicompat_encoding", econv_s_asciicompat_encoding, 1);
4468 rb_define_singleton_method(rb_cEncodingConverter, "search_convpath", econv_s_search_convpath, -1);
4469 rb_define_method(rb_cEncodingConverter, "initialize", econv_init, -1);
4470 rb_define_method(rb_cEncodingConverter, "inspect", econv_inspect, 0);
4471 rb_define_method(rb_cEncodingConverter, "convpath", econv_convpath, 0);
4472 rb_define_method(rb_cEncodingConverter, "source_encoding", econv_source_encoding, 0);
4473 rb_define_method(rb_cEncodingConverter, "destination_encoding", econv_destination_encoding, 0);
4474 rb_define_method(rb_cEncodingConverter, "primitive_convert", econv_primitive_convert, -1);
4475 rb_define_method(rb_cEncodingConverter, "convert", econv_convert, 1);
4476 rb_define_method(rb_cEncodingConverter, "finish", econv_finish, 0);
4477 rb_define_method(rb_cEncodingConverter, "primitive_errinfo", econv_primitive_errinfo, 0);
4478 rb_define_method(rb_cEncodingConverter, "insert_output", econv_insert_output, 1);
4479 rb_define_method(rb_cEncodingConverter, "putback", econv_putback, -1);
4480 rb_define_method(rb_cEncodingConverter, "last_error", econv_last_error, 0);
4481 rb_define_method(rb_cEncodingConverter, "replacement", econv_get_replacement, 0);
4482 rb_define_method(rb_cEncodingConverter, "replacement=", econv_set_replacement, 1);
4483 rb_define_method(rb_cEncodingConverter, "==", econv_equal, 1);
4484
4485 /* Document-const: INVALID_MASK
4486 *
4487 * Mask for invalid byte sequences
4488 */
4489 rb_define_const(rb_cEncodingConverter, "INVALID_MASK", INT2FIX(ECONV_INVALID_MASK));
4490
4491 /* Document-const: INVALID_REPLACE
4492 *
4493 * Replace invalid byte sequences
4494 */
4495 rb_define_const(rb_cEncodingConverter, "INVALID_REPLACE", INT2FIX(ECONV_INVALID_REPLACE));
4496
4497 /* Document-const: UNDEF_MASK
4498 *
4499 * Mask for a valid character in the source encoding but no related
4500 * character(s) in destination encoding.
4501 */
4502 rb_define_const(rb_cEncodingConverter, "UNDEF_MASK", INT2FIX(ECONV_UNDEF_MASK));
4503
4504 /* Document-const: UNDEF_REPLACE
4505 *
4506 * Replace byte sequences that are undefined in the destination encoding.
4507 */
4508 rb_define_const(rb_cEncodingConverter, "UNDEF_REPLACE", INT2FIX(ECONV_UNDEF_REPLACE));
4509
4510 /* Document-const: UNDEF_HEX_CHARREF
4511 *
4512 * Replace byte sequences that are undefined in the destination encoding
4513 * with an XML hexadecimal character reference. This is valid for XML
4514 * conversion.
4515 */
4516 rb_define_const(rb_cEncodingConverter, "UNDEF_HEX_CHARREF", INT2FIX(ECONV_UNDEF_HEX_CHARREF));
4517
4518 /* Document-const: PARTIAL_INPUT
4519 *
4520 * Indicates the source may be part of a larger string. See
4521 * primitive_convert for an example.
4522 */
4523 rb_define_const(rb_cEncodingConverter, "PARTIAL_INPUT", INT2FIX(ECONV_PARTIAL_INPUT));
4524
4525 /* Document-const: AFTER_OUTPUT
4526 *
4527 * Stop converting after some output is complete but before all of the
4528 * input was consumed. See primitive_convert for an example.
4529 */
4530 rb_define_const(rb_cEncodingConverter, "AFTER_OUTPUT", INT2FIX(ECONV_AFTER_OUTPUT));
4531
4532 /* Document-const: UNIVERSAL_NEWLINE_DECORATOR
4533 *
4534 * Decorator for converting CRLF and CR to LF
4535 */
4536 rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECORATOR", INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECORATOR));
4537
4538 /* Document-const: CRLF_NEWLINE_DECORATOR
4539 *
4540 * Decorator for converting LF to CRLF
4541 */
4542 rb_define_const(rb_cEncodingConverter, "CRLF_NEWLINE_DECORATOR", INT2FIX(ECONV_CRLF_NEWLINE_DECORATOR));
4543
4544 /* Document-const: CR_NEWLINE_DECORATOR
4545 *
4546 * Decorator for converting LF to CR
4547 */
4548 rb_define_const(rb_cEncodingConverter, "CR_NEWLINE_DECORATOR", INT2FIX(ECONV_CR_NEWLINE_DECORATOR));
4549
4550 /* Document-const: XML_TEXT_DECORATOR
4551 *
4552 * Escape as XML CharData
4553 */
4554 rb_define_const(rb_cEncodingConverter, "XML_TEXT_DECORATOR", INT2FIX(ECONV_XML_TEXT_DECORATOR));
4555
4556 /* Document-const: XML_ATTR_CONTENT_DECORATOR
4557 *
4558 * Escape as XML AttValue
4559 */
4560 rb_define_const(rb_cEncodingConverter, "XML_ATTR_CONTENT_DECORATOR", INT2FIX(ECONV_XML_ATTR_CONTENT_DECORATOR));
4561
4562 /* Document-const: XML_ATTR_QUOTE_DECORATOR
4563 *
4564 * Escape as XML AttValue
4565 */
4566 rb_define_const(rb_cEncodingConverter, "XML_ATTR_QUOTE_DECORATOR", INT2FIX(ECONV_XML_ATTR_QUOTE_DECORATOR));
4567
4568 rb_define_method(rb_eUndefinedConversionError, "source_encoding_name", ecerr_source_encoding_name, 0);
4569 rb_define_method(rb_eUndefinedConversionError, "destination_encoding_name", ecerr_destination_encoding_name, 0);
4570 rb_define_method(rb_eUndefinedConversionError, "source_encoding", ecerr_source_encoding, 0);
4571 rb_define_method(rb_eUndefinedConversionError, "destination_encoding", ecerr_destination_encoding, 0);
4572 rb_define_method(rb_eUndefinedConversionError, "error_char", ecerr_error_char, 0);
4573
4574 rb_define_method(rb_eInvalidByteSequenceError, "source_encoding_name", ecerr_source_encoding_name, 0);
4575 rb_define_method(rb_eInvalidByteSequenceError, "destination_encoding_name", ecerr_destination_encoding_name, 0);
4576 rb_define_method(rb_eInvalidByteSequenceError, "source_encoding", ecerr_source_encoding, 0);
4577 rb_define_method(rb_eInvalidByteSequenceError, "destination_encoding", ecerr_destination_encoding, 0);
4578 rb_define_method(rb_eInvalidByteSequenceError, "error_bytes", ecerr_error_bytes, 0);
4579 rb_define_method(rb_eInvalidByteSequenceError, "readagain_bytes", ecerr_readagain_bytes, 0);
4580 rb_define_method(rb_eInvalidByteSequenceError, "incomplete_input?", ecerr_incomplete_input, 0);
4581
4582 Init_newline();
4583 }
4584