12#include "ruby/internal/config.h"
17#include "internal/array.h"
18#include "internal/inits.h"
19#include "internal/object.h"
20#include "internal/string.h"
21#include "internal/transcode.h"
24#include "transcode_data.h"
27#define ENABLE_ECONV_NEWLINE_OPTION 1
30static VALUE rb_eUndefinedConversionError;
31static VALUE rb_eInvalidByteSequenceError;
32static VALUE rb_eConverterNotFoundError;
34VALUE rb_cEncodingConverter;
36static ID id_destination_encoding;
37static ID id_destination_encoding_name;
38static ID id_error_bytes;
39static ID id_error_char;
40static ID id_incomplete_input;
41static ID id_readagain_bytes;
42static ID id_source_encoding;
43static ID id_source_encoding_name;
45static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
46static VALUE sym_xml, sym_text, sym_attr;
47static VALUE sym_universal_newline;
48static VALUE sym_crlf_newline;
49static VALUE sym_cr_newline;
50static VALUE sym_lf_newline;
51#ifdef ENABLE_ECONV_NEWLINE_OPTION
52static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
54static VALUE sym_partial_input;
56static VALUE sym_invalid_byte_sequence;
57static VALUE sym_undefined_conversion;
58static VALUE sym_destination_buffer_full;
59static VALUE sym_source_buffer_empty;
60static VALUE sym_finished;
61static VALUE sym_after_output;
62static VALUE sym_incomplete_input;
65allocate_converted_string(
const char *sname,
const char *dname,
66 const unsigned char *str,
size_t len,
67 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
78 unsigned int next_table;
80 unsigned char next_byte;
81 unsigned int output_index;
83 ssize_t recognized_len;
84 ssize_t readagain_len;
99 char ary[
sizeof(double) >
sizeof(
void*) ?
sizeof(double) :
sizeof(
void*)];
100 double dummy_for_alignment;
103#define TRANSCODING_READBUF(tc) \
104 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
105 (tc)->readbuf.ary : \
107#define TRANSCODING_WRITEBUF(tc) \
108 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
109 (tc)->writebuf.ary : \
111#define TRANSCODING_WRITEBUF_SIZE(tc) \
112 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
113 sizeof((tc)->writebuf.ary) : \
114 (size_t)(tc)->transcoder->max_output)
115#define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
116#define TRANSCODING_STATE(tc) \
117 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
123 unsigned char *out_buf_start;
124 unsigned char *out_data_start;
125 unsigned char *out_data_end;
126 unsigned char *out_buf_end;
134 const char *source_encoding_name;
135 const char *destination_encoding_name;
137 const unsigned char *replacement_str;
138 size_t replacement_len;
139 const char *replacement_enc;
141 unsigned char *in_buf_start;
142 unsigned char *in_data_start;
143 unsigned char *in_data_end;
144 unsigned char *in_buf_end;
146 int replacement_allocated;
156 const char *source_encoding;
157 const char *destination_encoding;
158 const unsigned char *error_bytes_start;
159 size_t error_bytes_len;
160 size_t readagain_len;
173#define DECORATOR_P(sname, dname) (*(sname) == '\0')
185make_transcoder_entry(
const char *sname,
const char *dname)
190 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
191 val = (st_data_t)st_init_strcasetable();
192 st_add_direct(transcoder_table, (st_data_t)sname, val);
195 if (!st_lookup(table2, (st_data_t)dname, &val)) {
197 entry->sname = sname;
198 entry->dname = dname;
200 entry->transcoder = NULL;
201 val = (st_data_t)entry;
202 st_add_direct(table2, (st_data_t)dname, val);
208get_transcoder_entry(
const char *sname,
const char *dname)
213 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
217 if (!st_lookup(table2, (st_data_t)dname, &val)) {
226 const char *
const sname =
tr->src_encoding;
227 const char *
const dname =
tr->dst_encoding;
231 entry = make_transcoder_entry(sname, dname);
232 if (entry->transcoder) {
237 entry->transcoder =
tr;
241declare_transcoder(
const char *sname,
const char *dname,
const char *lib)
245 entry = make_transcoder_entry(sname, dname);
249static const char transcoder_lib_prefix[] =
"enc/trans/";
252rb_declare_transcoder(
const char *enc1,
const char *enc2,
const char *lib)
257 declare_transcoder(enc1, enc2, lib);
260#define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
271 const char *base_enc;
275transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
277 const char *dname = (
const char *)key;
281 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
288 *bfs->queue_last_ptr = q;
289 bfs->queue_last_ptr = &q->next;
291 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
296transcode_search_path(
const char *sname,
const char *dname,
297 void (*callback)(
const char *sname,
const char *dname,
int depth,
void *arg),
307 if (encoding_equal(sname, dname))
313 bfs.queue_last_ptr = &q->next;
316 bfs.visited = st_init_strcasetable();
317 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
323 bfs.queue_last_ptr = &bfs.queue;
325 if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) {
331 if (st_lookup(table2, (st_data_t)dname, &val)) {
332 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
338 bfs.base_enc = q->enc;
339 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
354 const char *enc = dname;
358 st_lookup(bfs.visited, (st_data_t)enc, &val);
362 enc = (
const char *)val;
367 st_lookup(bfs.visited, (st_data_t)enc, &val);
370 callback((
const char *)val, enc, --depth, arg);
371 enc = (
const char *)val;
375 st_free_table(bfs.visited);
380int rb_require_internal_silent(
VALUE fname);
385 if (entry->transcoder)
386 return entry->transcoder;
389 const char *
const lib = entry->lib;
390 const size_t len = strlen(lib);
391 const size_t total_len =
sizeof(transcoder_lib_prefix) - 1 + len;
393 char *
const path = RSTRING_PTR(fn);
395 memcpy(path, transcoder_lib_prefix,
sizeof(transcoder_lib_prefix) - 1);
396 memcpy(path +
sizeof(transcoder_lib_prefix) - 1, lib, len);
399 rb_require_internal_silent(fn);
402 if (entry->transcoder)
403 return entry->transcoder;
409get_replacement_character(
const char *encname,
size_t *len_ret,
const char **repl_encname_ptr)
411 if (encoding_equal(encname,
"UTF-8")) {
413 *repl_encname_ptr =
"UTF-8";
414 return "\xEF\xBF\xBD";
418 *repl_encname_ptr =
"US-ASCII";
427static const unsigned char *
429 const unsigned char *in_start,
430 const unsigned char *inchar_start,
431 const unsigned char *in_p,
432 size_t *char_len_ptr)
434 const unsigned char *ptr;
435 if (inchar_start - in_start < tc->recognized_len) {
436 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
437 inchar_start,
unsigned char, in_p - inchar_start);
438 ptr = TRANSCODING_READBUF(tc);
441 ptr = inchar_start - tc->recognized_len;
443 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
448transcode_restartable0(
const unsigned char **in_pos,
unsigned char **out_pos,
449 const unsigned char *in_stop,
unsigned char *out_stop,
454 int unitlen =
tr->input_unit_length;
455 ssize_t readagain_len = 0;
457 const unsigned char *inchar_start;
458 const unsigned char *in_p;
460 unsigned char *out_p;
462 in_p = inchar_start = *in_pos;
466#define SUSPEND(ret, num) \
468 tc->resume_position = (num); \
469 if (0 < in_p - inchar_start) \
470 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
471 inchar_start, unsigned char, in_p - inchar_start); \
474 tc->recognized_len += in_p - inchar_start; \
475 if (readagain_len) { \
476 tc->recognized_len -= readagain_len; \
477 tc->readagain_len = readagain_len; \
480 resume_label ## num:; \
482#define SUSPEND_OBUF(num) \
484 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
487#define SUSPEND_AFTER_OUTPUT(num) \
488 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
489 SUSPEND(econv_after_output, num); \
492#define next_table (tc->next_table)
493#define next_info (tc->next_info)
494#define next_byte (tc->next_byte)
495#define writebuf_len (tc->writebuf_len)
496#define writebuf_off (tc->writebuf_off)
498 switch (tc->resume_position) {
500 case 1:
goto resume_label1;
501 case 2:
goto resume_label2;
502 case 3:
goto resume_label3;
503 case 4:
goto resume_label4;
504 case 5:
goto resume_label5;
505 case 6:
goto resume_label6;
506 case 7:
goto resume_label7;
507 case 8:
goto resume_label8;
508 case 9:
goto resume_label9;
509 case 10:
goto resume_label10;
510 case 11:
goto resume_label11;
511 case 12:
goto resume_label12;
512 case 13:
goto resume_label13;
513 case 14:
goto resume_label14;
514 case 15:
goto resume_label15;
515 case 16:
goto resume_label16;
516 case 17:
goto resume_label17;
517 case 18:
goto resume_label18;
518 case 19:
goto resume_label19;
519 case 20:
goto resume_label20;
520 case 21:
goto resume_label21;
521 case 22:
goto resume_label22;
522 case 23:
goto resume_label23;
523 case 24:
goto resume_label24;
524 case 25:
goto resume_label25;
525 case 26:
goto resume_label26;
526 case 27:
goto resume_label27;
527 case 28:
goto resume_label28;
528 case 29:
goto resume_label29;
529 case 30:
goto resume_label30;
530 case 31:
goto resume_label31;
531 case 32:
goto resume_label32;
532 case 33:
goto resume_label33;
533 case 34:
goto resume_label34;
538 tc->recognized_len = 0;
539 next_table =
tr->conv_tree_start;
541 SUSPEND_AFTER_OUTPUT(24);
543 if (in_stop <= in_p) {
550#define BYTE_ADDR(index) (tr->byte_array + (index))
551#define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
552#define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
553#define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
554#define BL_MIN_BYTE (BL_BASE[0])
555#define BL_MAX_BYTE (BL_BASE[1])
556#define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
557#define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
559 next_byte = (
unsigned char)*in_p++;
561 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
564 next_info = (
VALUE)BL_ACTION(next_byte);
567 switch (next_info & 0x1F) {
570 const unsigned char *p = inchar_start;
573 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (
unsigned char)*p++;
575 writebuf_len = writebuf_off;
577 while (writebuf_off < writebuf_len) {
579 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
583 case 0x00:
case 0x04:
case 0x08:
case 0x0C:
584 case 0x10:
case 0x14:
case 0x18:
case 0x1C:
585 SUSPEND_AFTER_OUTPUT(25);
586 while (in_p >= in_stop) {
591 next_byte = (
unsigned char)*in_p++;
592 next_table = (
unsigned int)next_info;
597 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
600 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
601 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
604 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
605 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
606 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
609 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
610 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
611 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
612 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
615 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
616 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
617 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
618 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
621 tc->output_index = 0;
622 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
623 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
628 next_info = (
VALUE)(*
tr->func_ii)(TRANSCODING_STATE(tc), next_info);
632 const unsigned char *char_start;
634 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
635 next_info = (
VALUE)(*
tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
640 if (
tr->max_output <= out_stop - out_p)
641 out_p +=
tr->func_io(TRANSCODING_STATE(tc),
642 next_info, out_p, out_stop - out_p);
644 writebuf_len =
tr->func_io(TRANSCODING_STATE(tc),
646 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
648 while (writebuf_off < writebuf_len) {
650 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
656 const unsigned char *char_start;
659 if (
tr->max_output <= out_stop - out_p) {
660 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
661 out_p +=
tr->func_so(TRANSCODING_STATE(tc),
662 char_start, (
size_t)char_len,
663 out_p, out_stop - out_p);
666 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
667 writebuf_len =
tr->func_so(TRANSCODING_STATE(tc),
668 char_start, (
size_t)char_len,
669 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
671 while (writebuf_off < writebuf_len) {
673 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
680 const unsigned char *char_start;
683 if (
tr->max_output <= out_stop - out_p) {
684 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
685 out_p +=
tr->func_sio(TRANSCODING_STATE(tc),
686 char_start, (
size_t)char_len, next_info,
687 out_p, out_stop - out_p);
690 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
691 writebuf_len =
tr->func_sio(TRANSCODING_STATE(tc),
692 char_start, (
size_t)char_len, next_info,
693 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
695 while (writebuf_off < writebuf_len) {
697 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
703 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
704 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
705 SUSPEND_AFTER_OUTPUT(26);
706 while ((opt &
ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
710 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
714 in_p = inchar_start + (unitlen - tc->recognized_len);
720 invalid_len = tc->recognized_len + (in_p - inchar_start);
721 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
722 readagain_len = invalid_len - discard_len;
746 if (
tr->finish_func) {
748 if (
tr->max_output <= out_stop - out_p) {
749 out_p +=
tr->finish_func(TRANSCODING_STATE(tc),
750 out_p, out_stop - out_p);
753 writebuf_len =
tr->finish_func(TRANSCODING_STATE(tc),
754 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
756 while (writebuf_off < writebuf_len) {
758 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
773transcode_restartable(
const unsigned char **in_pos,
unsigned char **out_pos,
774 const unsigned char *in_stop,
unsigned char *out_stop,
778 if (tc->readagain_len) {
779 unsigned char *readagain_buf =
ALLOCA_N(
unsigned char, tc->readagain_len);
780 const unsigned char *readagain_pos = readagain_buf;
781 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
784 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
785 unsigned char, tc->readagain_len);
786 tc->readagain_len = 0;
787 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|
ECONV_PARTIAL_INPUT);
789 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
790 readagain_pos,
unsigned char, readagain_stop - readagain_pos);
791 tc->readagain_len += readagain_stop - readagain_pos;
795 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
806 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
808 if (
tr->state_init_func) {
809 (
tr->state_init_func)(TRANSCODING_STATE(tc));
811 tc->resume_position = 0;
812 tc->recognized_len = 0;
813 tc->readagain_len = 0;
814 tc->writebuf_len = 0;
815 tc->writebuf_off = 0;
816 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
817 tc->readbuf.ptr =
xmalloc(
tr->max_input);
819 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
820 tc->writebuf.ptr =
xmalloc(
tr->max_output);
827 const unsigned char **input_ptr,
const unsigned char *input_stop,
828 unsigned char **output_ptr,
unsigned char *output_stop,
831 return transcode_restartable(
832 input_ptr, output_ptr,
833 input_stop, output_stop,
841 if (
tr->state_fini_func) {
842 (
tr->state_fini_func)(TRANSCODING_STATE(tc));
844 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
845 xfree(tc->state.ptr);
846 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input)
847 xfree(tc->readbuf.ptr);
848 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output)
849 xfree(tc->writebuf.ptr);
859 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
860 size +=
tr->state_size;
862 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
863 size +=
tr->max_input;
865 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
866 size +=
tr->max_output;
872rb_econv_alloc(
int n_hint)
881 ec->source_encoding_name = NULL;
882 ec->destination_encoding_name = NULL;
884 ec->replacement_str = NULL;
885 ec->replacement_len = 0;
886 ec->replacement_enc = NULL;
887 ec->replacement_allocated = 0;
888 ec->in_buf_start = NULL;
889 ec->in_data_start = NULL;
890 ec->in_data_end = NULL;
891 ec->in_buf_end = NULL;
892 ec->num_allocated = n_hint;
895 ec->num_finished = 0;
898 ec->last_error.error_tc = NULL;
899 ec->last_error.source_encoding = NULL;
900 ec->last_error.destination_encoding = NULL;
901 ec->last_error.error_bytes_start = NULL;
902 ec->last_error.error_bytes_len = 0;
903 ec->last_error.readagain_len = 0;
904 ec->source_encoding = NULL;
905 ec->destination_encoding = NULL;
916 if (ec->num_trans == ec->num_allocated) {
917 n = ec->num_allocated * 2;
919 ec->num_allocated = n;
926 ec->elems[i].tc = rb_transcoding_open_by_transcoder(
tr, 0);
927 ec->elems[i].out_buf_start = p;
928 ec->elems[i].out_buf_end = p + bufsize;
929 ec->elems[i].out_data_start = p;
930 ec->elems[i].out_data_end = p;
935 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
936 for (j = ec->num_trans-1; i <= j; j--) {
939 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
954 for (i = 0; i < n; i++) {
956 tr = load_transcoder_entry(entries[i]);
961 ec = rb_econv_alloc(n);
963 for (i = 0; i < n; i++) {
965 ret = rb_econv_add_transcoder_at(ec,
tr, ec->num_trans);
981trans_open_i(
const char *sname,
const char *dname,
int depth,
void *arg)
985 if (!toarg->entries) {
988 toarg->entries[depth] = get_transcoder_entry(sname, dname);
992rb_econv_open0(
const char *sname,
const char *dname,
int ecflags)
1000 if (*sname) rb_enc_find_index(sname);
1001 if (*dname) rb_enc_find_index(dname);
1003 if (*sname ==
'\0' && *dname ==
'\0') {
1010 toarg.entries = NULL;
1011 toarg.num_additional = 0;
1012 num_trans = transcode_search_path(sname, dname, trans_open_i, (
void *)&toarg);
1013 entries = toarg.entries;
1014 if (num_trans < 0) {
1020 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1025 ec->flags = ecflags;
1026 ec->source_encoding_name = sname;
1027 ec->destination_encoding_name = dname;
1032#define MAX_ECFLAGS_DECORATORS 32
1035decorator_names(
int ecflags,
const char **decorators_ret)
1057 decorators_ret[num_decorators++] =
"xml_text_escape";
1059 decorators_ret[num_decorators++] =
"xml_attr_content_escape";
1061 decorators_ret[num_decorators++] =
"xml_attr_quote";
1064 decorators_ret[num_decorators++] =
"crlf_newline";
1066 decorators_ret[num_decorators++] =
"cr_newline";
1068 decorators_ret[num_decorators++] =
"lf_newline";
1070 decorators_ret[num_decorators++] =
"universal_newline";
1072 return num_decorators;
1080 const char *decorators[MAX_ECFLAGS_DECORATORS];
1083 num_decorators = decorator_names(ecflags, decorators);
1084 if (num_decorators == -1)
1091 for (i = 0; i < num_decorators; i++)
1097 ec->flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
1104 const unsigned char **input_ptr,
const unsigned char *input_stop,
1105 unsigned char **output_ptr,
unsigned char *output_stop,
1112 const unsigned char **ipp, *is, *iold;
1113 unsigned char **opp, *os, *oold;
1119 for (i = start; i < ec->num_trans; i++) {
1128 ipp = (
const unsigned char **)&prev_te->out_data_start;
1129 is = prev_te->out_data_end;
1132 if (i == ec->num_trans-1) {
1137 if (te->out_buf_start != te->out_data_start) {
1138 ssize_t len = te->out_data_end - te->out_data_start;
1139 ssize_t off = te->out_data_start - te->out_buf_start;
1140 MEMMOVE(te->out_buf_start, te->out_data_start,
unsigned char, len);
1141 te->out_data_start = te->out_buf_start;
1142 te->out_data_end -= off;
1144 opp = &te->out_data_end;
1145 os = te->out_buf_end;
1149 if (ec->num_finished != i)
1153 flags &= ~ECONV_AFTER_OUTPUT;
1156 f &= ~ECONV_AFTER_OUTPUT;
1159 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1160 if (iold != *ipp || oold != *opp)
1175 ec->num_finished = i+1;
1185 const unsigned char **input_ptr,
const unsigned char *input_stop,
1186 unsigned char **output_ptr,
unsigned char *output_stop,
1188 int *result_position_ptr)
1191 int needreport_index;
1194 unsigned char empty_buf;
1195 unsigned char *empty_ptr = &empty_buf;
1198 input_ptr = (
const unsigned char **)&empty_ptr;
1199 input_stop = empty_ptr;
1203 output_ptr = &empty_ptr;
1204 output_stop = empty_ptr;
1210 for (i = ec->num_trans-1; 0 <= i; i--) {
1211 switch (ec->elems[i].last_result) {
1218 goto found_needreport;
1225 rb_bug(
"unexpected transcode last result");
1235 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1237 result_position_ptr);
1249 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1250 sweep_start = needreport_index + 1;
1251 }
while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1253 for (i = ec->num_trans-1; 0 <= i; i--) {
1262 if (result_position_ptr)
1263 *result_position_ptr = i;
1267 if (result_position_ptr)
1268 *result_position_ptr = -1;
1274 const unsigned char **input_ptr,
const unsigned char *input_stop,
1275 unsigned char **output_ptr,
unsigned char *output_stop,
1279 int result_position;
1282 memset(&ec->last_error, 0,
sizeof(ec->last_error));
1284 if (ec->num_trans == 0) {
1286 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1287 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1288 len = output_stop - *output_ptr;
1289 memcpy(*output_ptr, ec->in_data_start, len);
1290 *output_ptr = output_stop;
1291 ec->in_data_start += len;
1295 len = ec->in_data_end - ec->in_data_start;
1296 memcpy(*output_ptr, ec->in_data_start, len);
1298 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1304 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1305 len = output_stop - *output_ptr;
1308 len = input_stop - *input_ptr;
1311 *(*output_ptr)++ = *(*input_ptr)++;
1315 memcpy(*output_ptr, *input_ptr, len);
1318 if (*input_ptr != input_stop)
1327 if (ec->elems[ec->num_trans-1].out_data_start) {
1328 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1329 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1330 if (data_start != data_end) {
1332 if (output_stop - *output_ptr < data_end - data_start) {
1333 len = output_stop - *output_ptr;
1334 memcpy(*output_ptr, data_start, len);
1335 *output_ptr = output_stop;
1336 ec->elems[ec->num_trans-1].out_data_start += len;
1340 len = data_end - data_start;
1341 memcpy(*output_ptr, data_start, len);
1343 ec->elems[ec->num_trans-1].out_data_start =
1344 ec->elems[ec->num_trans-1].out_data_end =
1345 ec->elems[ec->num_trans-1].out_buf_start;
1350 if (ec->in_buf_start &&
1351 ec->in_data_start != ec->in_data_end) {
1352 res = rb_trans_conv(ec, (
const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1360 *input_ptr != input_stop) {
1361 input_stop = *input_ptr;
1362 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1367 ec->num_trans == 1) {
1368 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1373 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1378 ec->last_error.result = res;
1383 ec->last_error.error_tc = error_tc;
1384 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1385 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1386 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1387 ec->last_error.error_bytes_len = error_tc->recognized_len;
1388 ec->last_error.readagain_len = error_tc->readagain_len;
1394static int output_replacement_character(
rb_econv_t *ec);
1400 unsigned char utfbuf[1024];
1401 const unsigned char *utf;
1403 int utf_allocated = 0;
1404 char charef_buf[16];
1405 const unsigned char *p;
1407 if (encoding_equal(ec->last_error.source_encoding,
"UTF-32BE")) {
1408 utf = ec->last_error.error_bytes_start;
1409 utf_len = ec->last_error.error_bytes_len;
1412 utf = allocate_converted_string(ec->last_error.source_encoding,
"UTF-32BE",
1413 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1414 utfbuf,
sizeof(utfbuf),
1418 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1422 if (utf_len % 4 != 0)
1426 while (4 <= utf_len) {
1432 snprintf(charef_buf,
sizeof(charef_buf),
"&#x%X;", u);
1454 const unsigned char **input_ptr,
const unsigned char *input_stop,
1455 unsigned char **output_ptr,
unsigned char *output_stop,
1460 unsigned char empty_buf;
1461 unsigned char *empty_ptr = &empty_buf;
1466 input_ptr = (
const unsigned char **)&empty_ptr;
1467 input_stop = empty_ptr;
1471 output_ptr = &empty_ptr;
1472 output_stop = empty_ptr;
1476 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1484 if (output_replacement_character(ec) == 0)
1495 if (output_replacement_character(ec) == 0)
1500 if (output_hex_charref(ec) == 0)
1518 tr = tc->transcoder;
1520 if (
tr->asciicompat_type == asciicompat_encoder)
1521 return tr->src_encoding;
1522 return tr->dst_encoding;
1525static unsigned char *
1526allocate_converted_string(
const char *sname,
const char *dname,
1527 const unsigned char *str,
size_t len,
1528 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
1529 size_t *dst_len_ptr)
1531 unsigned char *dst_str;
1538 const unsigned char *sp;
1542 dst_bufsize = caller_dst_bufsize;
1552 dst_str = caller_dst_buf;
1554 dst_str =
xmalloc(dst_bufsize);
1557 dp = dst_str+dst_len;
1559 dst_len = dp - dst_str;
1561 if (SIZE_MAX/2 < dst_bufsize) {
1565 if (dst_str == caller_dst_buf) {
1568 memcpy(tmp, dst_str, dst_bufsize/2);
1572 dst_str =
xrealloc(dst_str, dst_bufsize);
1574 dp = dst_str+dst_len;
1576 dst_len = dp - dst_str;
1582 *dst_len_ptr = dst_len;
1586 if (dst_str != caller_dst_buf)
1595 const unsigned char *str,
size_t len,
const char *str_encoding)
1598 unsigned char insert_buf[4096];
1599 const unsigned char *insert_str = NULL;
1602 int last_trans_index;
1605 unsigned char **buf_start_p;
1606 unsigned char **data_start_p;
1607 unsigned char **data_end_p;
1608 unsigned char **buf_end_p;
1617 if (encoding_equal(insert_encoding, str_encoding)) {
1622 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1623 str, len, insert_buf,
sizeof(insert_buf), &insert_len);
1624 if (insert_str == NULL)
1630 last_trans_index = ec->num_trans-1;
1631 if (ec->num_trans == 0) {
1633 buf_start_p = &ec->in_buf_start;
1634 data_start_p = &ec->in_data_start;
1635 data_end_p = &ec->in_data_end;
1636 buf_end_p = &ec->in_buf_end;
1638 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1639 tc = ec->elems[last_trans_index].tc;
1640 need += tc->readagain_len;
1641 if (need < insert_len)
1643 if (last_trans_index == 0) {
1644 buf_start_p = &ec->in_buf_start;
1645 data_start_p = &ec->in_data_start;
1646 data_end_p = &ec->in_data_end;
1647 buf_end_p = &ec->in_buf_end;
1651 buf_start_p = &ee->out_buf_start;
1652 data_start_p = &ee->out_data_start;
1653 data_end_p = &ee->out_data_end;
1654 buf_end_p = &ee->out_buf_end;
1659 buf_start_p = &ee->out_buf_start;
1660 data_start_p = &ee->out_data_start;
1661 data_end_p = &ee->out_data_end;
1662 buf_end_p = &ee->out_buf_end;
1663 tc = ec->elems[last_trans_index].tc;
1666 if (*buf_start_p == NULL) {
1667 unsigned char *buf =
xmalloc(need);
1669 *data_start_p = buf;
1671 *buf_end_p = buf+need;
1673 else if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1674 MEMMOVE(*buf_start_p, *data_start_p,
unsigned char, *data_end_p - *data_start_p);
1675 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1676 *data_start_p = *buf_start_p;
1677 if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1679 size_t s = (*data_end_p - *buf_start_p) + need;
1683 *data_start_p = buf;
1684 *data_end_p = buf + (*data_end_p - *buf_start_p);
1686 *buf_end_p = buf + s;
1690 memcpy(*data_end_p, insert_str, insert_len);
1691 *data_end_p += insert_len;
1692 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1693 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1694 *data_end_p += tc->readagain_len;
1695 tc->readagain_len = 0;
1698 if (insert_str != str && insert_str != insert_buf)
1699 xfree((
void*)insert_str);
1703 if (insert_str != str && insert_str != insert_buf)
1704 xfree((
void*)insert_str);
1713 if (ec->replacement_allocated) {
1714 xfree((
void *)ec->replacement_str);
1716 for (i = 0; i < ec->num_trans; i++) {
1717 rb_transcoding_close(ec->elems[i].tc);
1718 if (ec->elems[i].out_buf_start)
1719 xfree(ec->elems[i].out_buf_start);
1721 xfree(ec->in_buf_start);
1732 if (ec->replacement_allocated) {
1733 size += ec->replacement_len;
1735 for (i = 0; i < ec->num_trans; i++) {
1736 size += rb_transcoding_memsize(ec->elems[i].tc);
1738 if (ec->elems[i].out_buf_start) {
1739 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1742 size += ec->in_buf_end - ec->in_buf_start;
1751 if (ec->num_trans == 0)
1753#if SIZEOF_SIZE_T > SIZEOF_INT
1754 if (ec->elems[0].tc->readagain_len > INT_MAX)
return INT_MAX;
1756 return (
int)ec->elems[0].tc->readagain_len;
1763 if (ec->num_trans == 0 || n == 0)
1765 tc = ec->elems[0].tc;
1766 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1767 tc->readagain_len -= n;
1771 const char *ascii_compat_name;
1772 const char *ascii_incompat_name;
1776asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1782 if (DECORATOR_P(entry->sname, entry->dname))
1784 tr = load_transcoder_entry(entry);
1785 if (
tr &&
tr->asciicompat_type == asciicompat_decoder) {
1786 data->ascii_compat_name =
tr->dst_encoding;
1799 if (!st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v))
1810 if (table2->num_entries != 1)
1813 data.ascii_incompat_name = ascii_incompat_name;
1814 data.ascii_compat_name = NULL;
1815 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1816 return data.ascii_compat_name;
1828 unsigned const char *sp, *se;
1829 unsigned char *ds, *dp, *de;
1838 rb_enc_associate(dst, dst_enc);
1843 dst_enc = rb_enc_get(dst);
1848 max_output = ec->last_tc->transcoder->max_output;
1854 long dlen = RSTRING_LEN(dst);
1856 unsigned long new_capa = (
unsigned long)dlen + len + max_output;
1857 if (LONG_MAX < new_capa)
1861 sp = (
const unsigned char *)ss;
1863 ds = (
unsigned char *)RSTRING_PTR(dst);
1867 switch (coderange) {
1870 cr = (int)coderange;
1879 len -= (
const char *)sp - ss;
1880 ss = (
const char *)sp;
1916rb_econv_add_converter(
rb_econv_t *ec,
const char *sname,
const char *dname,
int n)
1921 if (ec->started != 0)
1924 entry = get_transcoder_entry(sname, dname);
1928 tr = load_transcoder_entry(entry);
1931 return rb_econv_add_transcoder_at(ec,
tr, n);
1935rb_econv_decorate_at(
rb_econv_t *ec,
const char *decorator_name,
int n)
1937 return rb_econv_add_converter(ec,
"", decorator_name, n);
1945 if (ec->num_trans == 0)
1946 return rb_econv_decorate_at(ec, decorator_name, 0);
1948 tr = ec->elems[0].tc->transcoder;
1950 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1951 tr->asciicompat_type == asciicompat_decoder)
1952 return rb_econv_decorate_at(ec, decorator_name, 1);
1954 return rb_econv_decorate_at(ec, decorator_name, 0);
1962 if (ec->num_trans == 0)
1963 return rb_econv_decorate_at(ec, decorator_name, 0);
1965 tr = ec->elems[ec->num_trans-1].tc->transcoder;
1967 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1968 tr->asciicompat_type == asciicompat_encoder)
1969 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
1971 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
1977 const char *dname = 0;
1981 dname =
"universal_newline";
1984 dname =
"crlf_newline";
1987 dname =
"cr_newline";
1990 dname =
"lf_newline";
1995 const rb_transcoder *transcoder = get_transcoder_entry(
"", dname)->transcoder;
1996 int num_trans = ec->num_trans;
1999 for (i=0; i < num_trans; i++) {
2000 if (transcoder == ec->elems[i].tc->transcoder) {
2001 rb_transcoding_close(ec->elems[i].tc);
2002 xfree(ec->elems[i].out_buf_start);
2006 ec->elems[j++] = ec->elems[i];
2010 ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2014econv_description(
const char *sname,
const char *dname,
int ecflags,
VALUE mesg)
2016 int has_description = 0;
2021 if (*sname !=
'\0' || *dname !=
'\0') {
2024 else if (*dname ==
'\0')
2028 has_description = 1;
2035 const char *pre =
"";
2036 if (has_description)
2066 has_description = 1;
2068 if (!has_description) {
2080 econv_description(sname, dname, ecflags, mesg);
2082 exc =
rb_exc_new3(rb_eConverterNotFoundError, mesg);
2092 const char *err = (
const char *)ec->last_error.error_bytes_start;
2093 size_t error_len = ec->last_error.error_bytes_len;
2096 size_t readagain_len = ec->last_error.readagain_len;
2102 ec->last_error.source_encoding);
2104 else if (readagain_len) {
2105 bytes2 =
rb_str_new(err+error_len, readagain_len);
2110 ec->last_error.source_encoding);
2115 ec->last_error.source_encoding);
2118 exc =
rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2125 VALUE bytes =
rb_str_new((
const char *)ec->last_error.error_bytes_start,
2126 ec->last_error.error_bytes_len);
2129 if (strcmp(ec->last_error.source_encoding,
"UTF-8") == 0) {
2131 const char *start, *end;
2133 start = (
const char *)ec->last_error.error_bytes_start;
2134 end = start + ec->last_error.error_bytes_len;
2135 n = rb_enc_precise_mbclen(start, end, utf8);
2138 unsigned int cc = rb_enc_mbc_to_codepoint(start, end, utf8);
2144 if (strcmp(ec->last_error.source_encoding,
2145 ec->source_encoding_name) == 0 &&
2146 strcmp(ec->last_error.destination_encoding,
2147 ec->destination_encoding_name) == 0) {
2150 ec->last_error.source_encoding,
2151 ec->last_error.destination_encoding);
2155 mesg =
rb_sprintf(
"%s to %s in conversion from %s",
2157 ec->last_error.destination_encoding,
2158 ec->source_encoding_name);
2159 for (i = 0; i < ec->num_trans; i++) {
2161 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
2163 ec->elems[i].tc->transcoder->dst_encoding);
2166 exc =
rb_exc_new3(rb_eUndefinedConversionError, mesg);
2167 idx = rb_enc_find_index(ec->last_error.source_encoding);
2169 rb_enc_associate_index(bytes, idx);
2178 int idx = rb_enc_find_index(ec->last_error.source_encoding);
2180 rb_ivar_set(exc, id_source_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2181 idx = rb_enc_find_index(ec->last_error.destination_encoding);
2183 rb_ivar_set(exc, id_destination_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2190 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2192 unsigned char **out_start_ptr,
2193 unsigned char **out_pos,
2194 unsigned char **out_stop_ptr)
2196 size_t len = (*out_pos - *out_start_ptr);
2197 size_t new_len = (len + max_output) * 2;
2198 *out_start_ptr = resize_destination(destination, len, new_len);
2199 *out_pos = *out_start_ptr + len;
2200 *out_stop_ptr = *out_start_ptr + new_len;
2208 const unsigned char *replacement;
2209 const char *repl_enc;
2210 const char *ins_enc;
2213 if (ec->replacement_str)
2220 tr = tc->transcoder;
2221 rb_enc_find(
tr->dst_encoding);
2222 replacement = (
const unsigned char *)get_replacement_character(ins_enc, &len, &repl_enc);
2225 replacement = (
unsigned char *)
"?";
2230 ec->replacement_str = replacement;
2231 ec->replacement_len = len;
2232 ec->replacement_enc = repl_enc;
2233 ec->replacement_allocated = 0;
2239 const unsigned char *str,
size_t len,
const char *encname)
2241 unsigned char *str2;
2243 const char *encname2;
2247 if (!*encname2 || encoding_equal(encname, encname2)) {
2249 MEMCPY(str2, str,
unsigned char, len);
2254 str2 = allocate_converted_string(encname, encname2, str, len, NULL, 0, &len2);
2259 if (ec->replacement_allocated) {
2260 xfree((
void *)ec->replacement_str);
2262 ec->replacement_allocated = 1;
2263 ec->replacement_str = str2;
2264 ec->replacement_len = len2;
2265 ec->replacement_enc = encname2;
2274 if (make_replacement(ec) == -1)
2285#define hash_fallback rb_hash_aref
2306transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2307 const unsigned char *in_stop,
unsigned char *out_stop,
2309 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2310 const char *src_encoding,
2311 const char *dst_encoding,
2318 unsigned char *out_start = *out_pos;
2328 if (!
NIL_P(ecopts) && RB_TYPE_P(ecopts,
T_HASH)) {
2329 fallback = rb_hash_aref(ecopts, sym_fallback);
2330 if (RB_TYPE_P(fallback,
T_HASH)) {
2331 fallback_func = hash_fallback;
2334 fallback_func = proc_fallback;
2337 fallback_func = method_fallback;
2340 fallback_func = aref_fallback;
2343 last_tc = ec->last_tc;
2344 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2350 VALUE rep = rb_enc_str_new(
2351 (
const char *)ec->last_error.error_bytes_start,
2352 ec->last_error.error_bytes_len,
2353 rb_enc_find(ec->last_error.source_encoding));
2354 rep = (*fallback_func)(fallback, rep);
2355 if (!UNDEF_P(rep) && !
NIL_P(rep)) {
2358 RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
2359 if ((
int)ret == -1) {
2369 exc = make_econv_exception(ec);
2375 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2385transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2386 const unsigned char *in_stop,
unsigned char *out_stop,
2388 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2389 const char *src_encoding,
2390 const char *dst_encoding,
2397 unsigned char *out_start = *out_pos;
2398 const unsigned char *ptr;
2406 last_tc = ec->last_tc;
2407 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2412 unsigned char input_byte;
2413 const unsigned char *p = &input_byte;
2416 if (ptr < in_stop) {
2427 if (&input_byte != p)
2428 ptr += p - &input_byte;
2433 exc = make_econv_exception(ec);
2439 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2460static unsigned char *
2461str_transcoding_resize(
VALUE destination,
size_t len,
size_t new_len)
2464 return (
unsigned char *)RSTRING_PTR(destination);
2468econv_opts(
VALUE opt,
int ecflags)
2471 int newlineflag = 0;
2473 v = rb_hash_aref(opt, sym_invalid);
2476 else if (v==sym_replace) {
2483 v = rb_hash_aref(opt, sym_undef);
2486 else if (v==sym_replace) {
2493 v = rb_hash_aref(opt, sym_replace);
2498 v = rb_hash_aref(opt, sym_xml);
2503 else if (v==sym_attr) {
2514#ifdef ENABLE_ECONV_NEWLINE_OPTION
2515 v = rb_hash_aref(opt, sym_newline);
2518 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2519 if (v == sym_universal) {
2522 else if (v == sym_crlf) {
2525 else if (v == sym_cr) {
2528 else if (v == sym_lf) {
2543 v = rb_hash_aref(opt, sym_universal_newline);
2546 newlineflag |= !
NIL_P(v);
2548 v = rb_hash_aref(opt, sym_crlf_newline);
2551 newlineflag |= !
NIL_P(v);
2553 v = rb_hash_aref(opt, sym_cr_newline);
2556 newlineflag |= !
NIL_P(v);
2558 v = rb_hash_aref(opt, sym_lf_newline);
2561 newlineflag |= !
NIL_P(v);
2563 switch (newlineflag) {
2565 ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2566 ecflags |= setflags;
2570 rb_warning(
":newline option precedes other newline options");
2584 if (
NIL_P(opthash)) {
2588 ecflags = econv_opts(opthash, ecflags);
2590 v = rb_hash_aref(opthash, sym_replace);
2593 if (is_broken_string(v)) {
2597 rb_enc_name(rb_enc_get(v)));
2600 newhash = rb_hash_new();
2601 rb_hash_aset(newhash, sym_replace, v);
2604 v = rb_hash_aref(opthash, sym_fallback);
2606 VALUE h = rb_check_hash_type(v);
2611 newhash = rb_hash_new();
2612 rb_hash_aset(newhash, sym_fallback, v);
2616 if (!
NIL_P(newhash))
2617 rb_hash_freeze(newhash);
2635 if (
NIL_P(opthash)) {
2640 rb_bug(
"rb_econv_open_opts called with invalid opthash");
2641 replacement = rb_hash_aref(opthash, sym_replace);
2644 ec =
rb_econv_open(source_encoding, destination_encoding, ecflags);
2648 if (!
NIL_P(replacement)) {
2653 (
const unsigned char *)RSTRING_PTR(replacement),
2654 RSTRING_LEN(replacement),
2672 if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) ||
2673 !(enc = rb_enc_from_index(encidx))) {
2679 n = rb_enc_name(enc);
2694 const char *sname, *dname;
2695 int sencidx, dencidx;
2697 dencidx = enc_arg(arg1, &dname, &denc);
2700 sencidx = rb_enc_get_index(str);
2701 senc = rb_enc_from_index(sencidx);
2702 sname = rb_enc_name(senc);
2705 sencidx = enc_arg(arg2, &sname, &senc);
2716str_transcode0(
int argc,
VALUE *argv,
VALUE *self,
int ecflags,
VALUE ecopts)
2722 unsigned char *buf, *bp, *sp;
2723 const unsigned char *fromp;
2725 const char *sname, *dname;
2727 int explicitly_invalid_replace = TRUE;
2729 rb_check_arity(argc, 0, 2);
2732 arg1 = rb_enc_default_internal();
2734 if (!ecflags)
return -1;
2735 arg1 = rb_obj_encoding(str);
2738 explicitly_invalid_replace = FALSE;
2745 arg2 = argc<=1 ?
Qnil : argv[1];
2746 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2752 if (senc && senc == denc) {
2755 if (!
NIL_P(ecopts)) {
2756 rep = rb_hash_aref(ecopts, sym_replace);
2758 dest = rb_enc_str_scrub(senc, str, rep);
2759 if (
NIL_P(dest)) dest = str;
2763 return NIL_P(arg2) ? -1 : dencidx;
2765 if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
2766 if (is_ascii_string(str)) {
2770 if (encoding_equal(sname, dname)) {
2771 return NIL_P(arg2) ? -1 : dencidx;
2775 if (senc && denc && !rb_enc_asciicompat(senc) && !rb_enc_asciicompat(denc)) {
2781 if (encoding_equal(sname, dname)) {
2787 fromp = sp = (
unsigned char *)RSTRING_PTR(str);
2788 slen = RSTRING_LEN(str);
2791 bp = (
unsigned char *)RSTRING_PTR(dest);
2793 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts);
2794 if (fromp != sp+slen) {
2797 buf = (
unsigned char *)RSTRING_PTR(dest);
2803 dencidx = rb_define_dummy_encoding(dname);
2813str_transcode(
int argc,
VALUE *argv,
VALUE *self)
2819 argc = rb_scan_args(argc, argv,
"02:", NULL, NULL, &opt);
2823 return str_transcode0(argc, argv, self, ecflags, ecopts);
2827str_encode_associate(
VALUE str,
int encidx)
2831 rb_enc_associate_index(str, encidx);
2834 if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
2854str_encode_bang(
int argc,
VALUE *argv,
VALUE str)
2862 encidx = str_transcode(argc, argv, &newstr);
2864 if (encidx < 0)
return str;
2865 if (newstr == str) {
2866 rb_enc_associate_index(str, encidx);
2870 return str_encode_associate(str, encidx);
2879 int encidx = str_transcode(argc, argv, &newstr);
2880 return encoded_dup(newstr, str, encidx);
2889 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2890 return encoded_dup(newstr, str, encidx);
2894encoded_dup(
VALUE newstr,
VALUE str,
int encidx)
2897 if (newstr == str) {
2899 rb_enc_associate_index(newstr, encidx);
2905 return str_encode_associate(newstr, encidx);
2914econv_free(
void *ptr)
2921econv_memsize(
const void *ptr)
2928 {0, econv_free, econv_memsize,},
2929 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
2933econv_s_allocate(
VALUE klass)
2939make_dummy_encoding(
const char *name)
2943 idx = rb_define_dummy_encoding(name);
2944 enc = rb_enc_from_index(idx);
2949make_encoding(
const char *name)
2952 enc = rb_enc_find(name);
2954 enc = make_dummy_encoding(name);
2959make_encobj(
const char *name)
2961 return rb_enc_from_encoding(make_encoding(name));
2983econv_s_asciicompat_encoding(
VALUE klass,
VALUE arg)
2985 const char *arg_name, *result_name;
2988 enc_arg(&arg, &arg_name, &arg_enc);
2992 if (result_name == NULL)
2995 result_enc = make_encoding(result_name);
2997 return rb_enc_from_encoding(result_enc);
3001econv_args(
int argc,
VALUE *argv,
3003 const char **sname_p,
const char **dname_p,
3008 VALUE opt, flags_v, ecopts;
3010 const char *sname, *dname;
3014 argc = rb_scan_args(argc, argv,
"21:", snamev_p, dnamev_p, &flags_v, &opt);
3016 if (!
NIL_P(flags_v)) {
3018 rb_error_arity(argc + 1, 2, 3);
3023 else if (!
NIL_P(opt)) {
3032 sidx = rb_to_encoding_index(*snamev_p);
3034 senc = rb_enc_from_index(sidx);
3041 didx = rb_to_encoding_index(*dnamev_p);
3043 denc = rb_enc_from_index(didx);
3056 *ecflags_p = ecflags;
3061decorate_convpath(
VALUE convpath,
int ecflags)
3064 const char *decorators[MAX_ECFLAGS_DECORATORS];
3068 num_decorators = decorator_names(ecflags, decorators);
3069 if (num_decorators == -1)
3072 len = n = RARRAY_LENINT(convpath);
3075 if (RB_TYPE_P(pair,
T_ARRAY)) {
3076 const char *sname = rb_enc_name(rb_to_encoding(
RARRAY_AREF(pair, 0)));
3077 const char *dname = rb_enc_name(rb_to_encoding(
RARRAY_AREF(pair, 1)));
3082 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
3083 tr->asciicompat_type == asciicompat_encoder) {
3085 rb_ary_store(convpath, len + num_decorators - 1, pair);
3089 rb_ary_store(convpath, len + num_decorators - 1, pair);
3093 for (i = 0; i < num_decorators; i++)
3100search_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3105 if (
NIL_P(*ary_p)) {
3106 *ary_p = rb_ary_new();
3109 if (DECORATOR_P(sname, dname)) {
3113 v = rb_assoc_new(make_encobj(sname), make_encobj(dname));
3115 rb_ary_store(*ary_p, depth, v);
3144econv_s_search_convpath(
int argc,
VALUE *argv,
VALUE klass)
3146 VALUE snamev, dnamev;
3147 const char *sname, *dname;
3153 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3156 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3158 if (
NIL_P(convpath)) {
3165 if (decorate_convpath(convpath, ecflags) == -1) {
3184 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3186 return RTEST(convpath);
3196rb_econv_init_by_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3204 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3211rb_econv_init_by_convpath(
VALUE self,
VALUE convpath,
3212 const char **sname_p,
const char **dname_p,
3220 const char *sname, *dname;
3222 ec = rb_econv_alloc(RARRAY_LENINT(convpath));
3226 VALUE snamev, dnamev;
3228 elt = rb_ary_entry(convpath, i);
3229 if (!
NIL_P(pair = rb_check_array_type(elt))) {
3232 snamev = rb_ary_entry(pair, 0);
3233 enc_arg(&snamev, &sname, &senc);
3234 dnamev = rb_ary_entry(pair, 1);
3235 enc_arg(&dnamev, &dname, &denc);
3241 if (DECORATOR_P(sname, dname)) {
3242 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3251 int j = ec->num_trans;
3254 arg.index = ec->num_trans;
3256 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3257 if (ret == -1 || arg.ret == -1) {
3258 VALUE msg =
rb_sprintf(
"adding conversion failed: %s to %s", sname, dname);
3266 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3269 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3280 ec->source_encoding_name = *sname_p;
3281 ec->destination_encoding_name = *dname_p;
3401 VALUE snamev, dnamev;
3402 const char *sname, *dname;
3412 if (argc == 1 && !
NIL_P(convpath = rb_check_array_type(argv[0]))) {
3413 ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
3418 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3429 if (!DECORATOR_P(sname, dname)) {
3431 senc = make_dummy_encoding(sname);
3433 denc = make_dummy_encoding(dname);
3438 ec->source_encoding = senc;
3439 ec->destination_encoding = denc;
3457econv_inspect(
VALUE self)
3464 return rb_sprintf(
"#<%s: uninitialized>", cname);
3466 const char *sname = ec->source_encoding_name;
3467 const char *dname = ec->destination_encoding_name;
3470 econv_description(sname, dname, ec->flags, str);
3477check_econv(
VALUE self)
3493 return rb_enc_from_encoding(encoding);
3503econv_source_encoding(
VALUE self)
3506 return econv_get_encoding(ec->source_encoding);
3516econv_destination_encoding(
VALUE self)
3519 return econv_get_encoding(ec->destination_encoding);
3545econv_convpath(
VALUE self)
3551 result = rb_ary_new();
3552 for (i = 0; i < ec->num_trans; i++) {
3555 if (DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
3558 v = rb_assoc_new(make_encobj(
tr->src_encoding), make_encobj(
tr->dst_encoding));
3559 rb_ary_push(result, v);
3580 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3581 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3583 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3584 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3586 if (ec1->flags != ec2->flags)
return Qfalse;
3587 if (ec1->replacement_enc != ec2->replacement_enc &&
3588 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3590 if (ec1->replacement_len != ec2->replacement_len)
return Qfalse;
3591 if (ec1->replacement_str != ec2->replacement_str &&
3592 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3595 if (ec1->num_trans != ec2->num_trans)
return Qfalse;
3596 for (i = 0; i < ec1->num_trans; i++) {
3597 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3713econv_primitive_convert(
int argc,
VALUE *argv,
VALUE self)
3715 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3718 const unsigned char *ip, *is;
3719 unsigned char *op, *os;
3720 long output_byteoffset, output_bytesize;
3721 unsigned long output_byteend;
3724 argc = rb_scan_args(argc, argv,
"23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3726 if (
NIL_P(output_byteoffset_v))
3727 output_byteoffset = 0;
3729 output_byteoffset =
NUM2LONG(output_byteoffset_v);
3731 if (
NIL_P(output_bytesize_v))
3732 output_bytesize = 0;
3734 output_bytesize =
NUM2LONG(output_bytesize_v);
3736 if (!
NIL_P(flags_v)) {
3738 rb_error_arity(argc + 1, 2, 5);
3742 else if (!
NIL_P(opt)) {
3745 v = rb_hash_aref(opt, sym_partial_input);
3748 v = rb_hash_aref(opt, sym_after_output);
3761 if (
NIL_P(output_bytesize_v)) {
3767 if (!
NIL_P(input) && output_bytesize < RSTRING_LEN(input))
3768 output_bytesize = RSTRING_LEN(input);
3773 if (
NIL_P(output_byteoffset_v))
3774 output_byteoffset = RSTRING_LEN(output);
3776 if (output_byteoffset < 0)
3779 if (RSTRING_LEN(output) < output_byteoffset)
3782 if (output_bytesize < 0)
3785 output_byteend = (
unsigned long)output_byteoffset +
3786 (
unsigned long)output_bytesize;
3788 if (output_byteend < (
unsigned long)output_byteoffset ||
3789 LONG_MAX < output_byteend)
3799 ip = (
const unsigned char *)RSTRING_PTR(input);
3800 is = ip + RSTRING_LEN(input);
3803 op = (
unsigned char *)RSTRING_PTR(output) + output_byteoffset;
3804 os = op + output_bytesize;
3808 if (!
NIL_P(input)) {
3813 if (LONG_MAX / 2 < output_bytesize)
3815 output_bytesize *= 2;
3816 output_byteoffset_v =
Qnil;
3820 if (ec->destination_encoding) {
3821 rb_enc_associate(output, ec->destination_encoding);
3824 return econv_result_to_symbol(res);
3862econv_convert(
VALUE self,
VALUE source_string)
3880 ret = econv_primitive_convert(ac, av, self);
3882 if (ret == sym_invalid_byte_sequence ||
3883 ret == sym_undefined_conversion ||
3884 ret == sym_incomplete_input) {
3885 VALUE exc = make_econv_exception(ec);
3889 if (ret == sym_finished) {
3893 if (ret != sym_source_buffer_empty) {
3894 rb_bug(
"unexpected result of econv_primitive_convert");
3912econv_finish(
VALUE self)
3928 ret = econv_primitive_convert(ac, av, self);
3930 if (ret == sym_invalid_byte_sequence ||
3931 ret == sym_undefined_conversion ||
3932 ret == sym_incomplete_input) {
3933 VALUE exc = make_econv_exception(ec);
3937 if (ret != sym_finished) {
3938 rb_bug(
"unexpected result of econv_primitive_convert");
4020econv_primitive_errinfo(
VALUE self)
4028 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4029 rb_ary_store(ary, 4,
Qnil);
4031 if (ec->last_error.source_encoding)
4032 rb_ary_store(ary, 1,
rb_str_new2(ec->last_error.source_encoding));
4034 if (ec->last_error.destination_encoding)
4035 rb_ary_store(ary, 2,
rb_str_new2(ec->last_error.destination_encoding));
4037 if (ec->last_error.error_bytes_start) {
4038 rb_ary_store(ary, 3,
rb_str_new((
const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4039 rb_ary_store(ary, 4,
rb_str_new((
const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4080 const char *insert_enc;
4088 string =
rb_str_encode(
string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0,
Qnil);
4090 ret =
rb_econv_insert_output(ec, (
const unsigned char *)RSTRING_PTR(
string), RSTRING_LEN(
string), insert_enc);
4123econv_putback(
int argc,
VALUE *argv,
VALUE self)
4130 if (!rb_check_arity(argc, 0, 1) ||
NIL_P(max = argv[0])) {
4136 if (putbackable < n)
4143 if (ec->source_encoding) {
4144 rb_enc_associate(str, ec->source_encoding);
4171econv_last_error(
VALUE self)
4176 exc = make_econv_exception(ec);
4195econv_get_replacement(
VALUE self)
4201 ret = make_replacement(ec);
4203 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4206 enc = rb_enc_find(ec->replacement_enc);
4207 return rb_enc_str_new((
const char *)ec->replacement_str, (
long)ec->replacement_len, enc);
4229 enc = rb_enc_get(
string);
4232 (
const unsigned char *)RSTRING_PTR(
string),
4233 RSTRING_LEN(
string),
4238 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4247 return make_econv_exception(ec);
4255 exc = make_econv_exception(ec);
4268ecerr_source_encoding_name(
VALUE self)
4270 return rb_attr_get(self, id_source_encoding_name);
4294ecerr_source_encoding(
VALUE self)
4306ecerr_destination_encoding_name(
VALUE self)
4308 return rb_attr_get(self, id_destination_encoding_name);
4318ecerr_destination_encoding(
VALUE self)
4320 return rb_attr_get(self, id_destination_encoding);
4339ecerr_error_char(
VALUE self)
4360ecerr_error_bytes(
VALUE self)
4372ecerr_readagain_bytes(
VALUE self)
4402ecerr_incomplete_input(
VALUE self)
4432 transcoder_table = st_init_strcasetable();
4434 id_destination_encoding = rb_intern_const(
"destination_encoding");
4435 id_destination_encoding_name = rb_intern_const(
"destination_encoding_name");
4436 id_error_bytes = rb_intern_const(
"error_bytes");
4437 id_error_char = rb_intern_const(
"error_char");
4438 id_incomplete_input = rb_intern_const(
"incomplete_input");
4439 id_readagain_bytes = rb_intern_const(
"readagain_bytes");
4440 id_source_encoding = rb_intern_const(
"source_encoding");
4441 id_source_encoding_name = rb_intern_const(
"source_encoding_name");
4443 sym_invalid =
ID2SYM(rb_intern_const(
"invalid"));
4444 sym_undef =
ID2SYM(rb_intern_const(
"undef"));
4445 sym_replace =
ID2SYM(rb_intern_const(
"replace"));
4446 sym_fallback =
ID2SYM(rb_intern_const(
"fallback"));
4447 sym_xml =
ID2SYM(rb_intern_const(
"xml"));
4448 sym_text =
ID2SYM(rb_intern_const(
"text"));
4449 sym_attr =
ID2SYM(rb_intern_const(
"attr"));
4451 sym_invalid_byte_sequence =
ID2SYM(rb_intern_const(
"invalid_byte_sequence"));
4452 sym_undefined_conversion =
ID2SYM(rb_intern_const(
"undefined_conversion"));
4453 sym_destination_buffer_full =
ID2SYM(rb_intern_const(
"destination_buffer_full"));
4454 sym_source_buffer_empty =
ID2SYM(rb_intern_const(
"source_buffer_empty"));
4455 sym_finished =
ID2SYM(rb_intern_const(
"finished"));
4456 sym_after_output =
ID2SYM(rb_intern_const(
"after_output"));
4457 sym_incomplete_input =
ID2SYM(rb_intern_const(
"incomplete_input"));
4458 sym_universal_newline =
ID2SYM(rb_intern_const(
"universal_newline"));
4459 sym_crlf_newline =
ID2SYM(rb_intern_const(
"crlf_newline"));
4460 sym_cr_newline =
ID2SYM(rb_intern_const(
"cr_newline"));
4461 sym_lf_newline =
ID2SYM(rb_intern(
"lf_newline"));
4462 sym_partial_input =
ID2SYM(rb_intern_const(
"partial_input"));
4464#ifdef ENABLE_ECONV_NEWLINE_OPTION
4465 sym_newline =
ID2SYM(rb_intern_const(
"newline"));
4466 sym_universal =
ID2SYM(rb_intern_const(
"universal"));
4467 sym_crlf =
ID2SYM(rb_intern_const(
"crlf"));
4468 sym_cr =
ID2SYM(rb_intern_const(
"cr"));
4469 sym_lf =
ID2SYM(rb_intern_const(
"lf"));
4476InitVM_transcode(
void)
4492 rb_define_method(rb_cEncodingConverter,
"source_encoding", econv_source_encoding, 0);
4493 rb_define_method(rb_cEncodingConverter,
"destination_encoding", econv_destination_encoding, 0);
4494 rb_define_method(rb_cEncodingConverter,
"primitive_convert", econv_primitive_convert, -1);
4497 rb_define_method(rb_cEncodingConverter,
"primitive_errinfo", econv_primitive_errinfo, 0);
4498 rb_define_method(rb_cEncodingConverter,
"insert_output", econv_insert_output, 1);
4500 rb_define_method(rb_cEncodingConverter,
"last_error", econv_last_error, 0);
4501 rb_define_method(rb_cEncodingConverter,
"replacement", econv_get_replacement, 0);
4502 rb_define_method(rb_cEncodingConverter,
"replacement=", econv_set_replacement, 1);
4594 rb_define_method(rb_eUndefinedConversionError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4595 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4596 rb_define_method(rb_eUndefinedConversionError,
"source_encoding", ecerr_source_encoding, 0);
4597 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding", ecerr_destination_encoding, 0);
4598 rb_define_method(rb_eUndefinedConversionError,
"error_char", ecerr_error_char, 0);
4600 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4601 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4602 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding", ecerr_source_encoding, 0);
4603 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding", ecerr_destination_encoding, 0);
4604 rb_define_method(rb_eInvalidByteSequenceError,
"error_bytes", ecerr_error_bytes, 0);
4605 rb_define_method(rb_eInvalidByteSequenceError,
"readagain_bytes", ecerr_readagain_bytes, 0);
4606 rb_define_method(rb_eInvalidByteSequenceError,
"incomplete_input?", ecerr_incomplete_input, 0);
ruby_coderange_type
What rb_enc_str_coderange() returns.
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
#define ECONV_XML_ATTR_QUOTE_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_QUOTE_DECORATOR.
#define ECONV_AFTER_OUTPUT
Old name of RUBY_ECONV_AFTER_OUTPUT.
#define rb_str_new2
Old name of rb_str_new_cstr.
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
#define ENC_CODERANGE_VALID
Old name of RUBY_ENC_CODERANGE_VALID.
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
Old name of RUBY_ECONV_UNIVERSAL_NEWLINE_DECORATOR.
#define REALLOC_N
Old name of RB_REALLOC_N.
#define ALLOC
Old name of RB_ALLOC.
#define xfree
Old name of ruby_xfree.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ECONV_XML_ATTR_CONTENT_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_CONTENT_DECORATOR.
#define rb_str_cat2
Old name of rb_str_cat_cstr.
#define ECONV_INVALID_MASK
Old name of RUBY_ECONV_INVALID_MASK.
#define ECONV_CRLF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CRLF_NEWLINE_DECORATOR.
#define xrealloc
Old name of ruby_xrealloc.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define ECONV_UNDEF_REPLACE
Old name of RUBY_ECONV_UNDEF_REPLACE.
#define ECONV_XML_TEXT_DECORATOR
Old name of RUBY_ECONV_XML_TEXT_DECORATOR.
#define rb_ary_new4
Old name of rb_ary_new_from_values.
#define ENC_CODERANGE_UNKNOWN
Old name of RUBY_ENC_CODERANGE_UNKNOWN.
#define ECONV_CR_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CR_NEWLINE_DECORATOR.
#define xmalloc
Old name of ruby_xmalloc.
#define ECONV_INVALID_REPLACE
Old name of RUBY_ECONV_INVALID_REPLACE.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
#define rb_exc_new3
Old name of rb_exc_new_str.
#define ECONV_UNDEF_MASK
Old name of RUBY_ECONV_UNDEF_MASK.
#define Qtrue
Old name of RUBY_Qtrue.
#define ECONV_PARTIAL_INPUT
Old name of RUBY_ECONV_PARTIAL_INPUT.
#define NUM2INT
Old name of RB_NUM2INT.
#define ECONV_ERROR_HANDLER_MASK
Old name of RUBY_ECONV_ERROR_HANDLER_MASK.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define ENC_CODERANGE_BROKEN
Old name of RUBY_ENC_CODERANGE_BROKEN.
#define ECONV_LF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_LF_NEWLINE_DECORATOR.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
#define ECONV_UNDEF_HEX_CHARREF
Old name of RUBY_ECONV_UNDEF_HEX_CHARREF.
#define NUM2LONG
Old name of RB_NUM2LONG.
#define ECONV_NEWLINE_DECORATOR_MASK
Old name of RUBY_ECONV_NEWLINE_DECORATOR_MASK.
#define rb_ary_new2
Old name of rb_ary_new_capa.
#define ENC_CODERANGE_SET(obj, cr)
Old name of RB_ENC_CODERANGE_SET.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
VALUE rb_eTypeError
TypeError exception.
VALUE rb_eRuntimeError
RuntimeError exception.
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
Identical to rb_typeddata_is_kind_of(), except it raises exceptions instead of returning false.
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
VALUE rb_eArgError
ArgumentError exception.
VALUE rb_eEncodingError
EncodingError exception.
void rb_warning(const char *fmt,...)
Issues a warning.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cEncoding
Encoding class.
VALUE rb_cString
String class.
VALUE rb_to_int(VALUE val)
Identical to rb_check_to_int(), except it raises in case of conversion mismatch.
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to)
Encoding conversion main routine.
int rb_enc_str_coderange(VALUE str)
Scans the passed string to collect its code range.
long rb_str_coderange_scan_restartable(const char *str, const char *end, rb_encoding *enc, int *cr)
Scans the passed string until it finds something odd.
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags)
Identical to rb_econv_prepare_opts(), except it additionally takes the initial value of flags.
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags)
Creates a rb_eConverterNotFoundError exception object (but does not raise).
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
Queries an encoding name which best suits for rb_econv_insert_output()'s last parameter.
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts)
Splits a keyword arguments hash (that for instance String#encode took) into a set of enum ruby_econv_...
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, int flags)
Converts a string from an encoding to another.
rb_econv_result_t
return value of rb_econv_convert()
@ econv_incomplete_input
The conversion stopped in middle of reading a character, possibly due to a partial read of a socket e...
@ econv_finished
The conversion stopped after converting everything.
@ econv_undefined_conversion
The conversion stopped when it found a character in the input which cannot be representable in the ou...
@ econv_after_output
The conversion stopped after writing something to somewhere, before reading everything.
@ econv_source_buffer_empty
The conversion stopped because there is no input.
@ econv_destination_buffer_full
The conversion stopped because there is no destination.
@ econv_invalid_byte_sequence
The conversion stopped when it found an invalid sequence.
int rb_econv_putbackable(rb_econv_t *ec)
Queries if rb_econv_putback() makes sense, i.e.
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
Queries if there is more than one way to convert between the passed two encodings.
rb_econv_t * rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags)
Creates a new instance of struct rb_econv_t.
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
Identical to rb_econv_str_convert(), except it appends the conversion result to the additionally pass...
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags)
Identical to rb_econv_str_append(), except it appends only a part of the passed string with conversio...
const char * rb_econv_asciicompat_encoding(const char *encname)
Queries the passed encoding's corresponding ASCII compatible encoding.
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
Appends the passed string to the passed converter's output buffer.
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
Identical to rb_econv_convert(), except it takes Ruby's string instead of C's pointer.
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts)
Identical to rb_econv_open(), except it additionally takes a hash of optional strings.
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
Identical to rb_econv_decorate_at_first(), except it adds to the opposite direction.
void rb_econv_binmode(rb_econv_t *ec)
This badly named function does not set the destination encoding to binary, but instead just nullifies...
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
"Decorate"s a converter.
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
Converts the contents of the passed string from its encoding to the passed one.
VALUE rb_econv_make_exception(rb_econv_t *ec)
This function makes sense right after rb_econv_convert() returns.
void rb_econv_check_error(rb_econv_t *ec)
This is a rb_econv_make_exception() + rb_exc_raise() combo.
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
Identical to rb_econv_str_convert(), except it converts only a part of the passed string.
void rb_econv_close(rb_econv_t *ec)
Destructs a converter.
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, long bytesize, VALUE dst, int flags)
Converts the passed C's pointer according to the passed converter, then append the conversion result ...
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
Puts back the bytes.
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
Assigns the replacement string.
VALUE rb_funcallv_public(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it only takes public methods into account.
#define rb_check_frozen
Just another name of rb_check_frozen.
VALUE rb_proc_call(VALUE recv, VALUE args)
Evaluates the passed proc with the passed arguments.
VALUE rb_obj_is_method(VALUE recv)
Queries if the given object is a method.
VALUE rb_method_call(int argc, const VALUE *argv, VALUE recv)
Evaluates the passed method with the passed arguments.
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
VALUE rb_str_tmp_new(long len)
Allocates a "temporary" string.
#define rb_str_new(str, len)
Allocates an instance of rb_cString.
void rb_str_shared_replace(VALUE dst, VALUE src)
Replaces the contents of the former with the latter.
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
VALUE rb_str_new_frozen(VALUE str)
Creates a frozen copy of the string, if necessary.
VALUE rb_str_dup(VALUE str)
Duplicates a string.
void rb_str_modify(VALUE str)
Declares that the string is about to be modified.
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
VALUE rb_str_resize(VALUE str, long len)
Overwrites the length of the string.
void rb_str_modify_expand(VALUE str, long capa)
Identical to rb_str_modify(), except it additionally expands the capacity of the receiver.
VALUE rb_str_dump(VALUE str)
"Inverse" of rb_eval_string().
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
VALUE rb_str_drop_bytes(VALUE str, long len)
Shrinks the given string for the given number of bytes.
VALUE rb_attr_get(VALUE obj, ID name)
Identical to rb_ivar_get()
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
VALUE rb_sym2str(VALUE id)
Identical to rb_id2str(), except it takes an instance of rb_cSymbol rather than an ID.
void rb_define_const(VALUE klass, const char *name, VALUE val)
Defines a Ruby level constant under a namespace.
VALUE rb_sprintf(const char *fmt,...)
Ruby's extended sprintf(3).
VALUE rb_str_catf(VALUE dst, const char *fmt,...)
Identical to rb_sprintf(), except it renders the output to the specified object rather than creating ...
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
#define ALLOCA_N(type, n)
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
#define DATA_PTR(obj)
Convenient getter macro.
@ RSTRING_EMBED_LEN_MAX
Max possible number of characters that can be embedded.
#define StringValue(v)
Ensures that the parameter object is a String.
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
#define InitVM(ext)
This macro is for internal use.
#define RTEST
This is an old name of RB_TEST.
uintptr_t VALUE
Type that represents a Ruby object.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.