Class: Encoding::Converter

Inherits:
Data show all
Defined in:
transcode.c,
transcode.c

Overview

Encoding conversion class.

Constant Summary collapse

INVALID_MASK =

Mask for invalid byte sequences

INT2FIX(ECONV_INVALID_MASK)
INVALID_REPLACE =

Replace invalid byte sequences

INT2FIX(ECONV_INVALID_REPLACE)
UNDEF_MASK =

Mask for a valid character in the source encoding but no related character(s) in destination encoding.

INT2FIX(ECONV_UNDEF_MASK)
UNDEF_REPLACE =

Replace byte sequences that are undefined in the destination encoding.

INT2FIX(ECONV_UNDEF_REPLACE)
UNDEF_HEX_CHARREF =

Replace byte sequences that are undefined in the destination encoding with an XML hexadecimal character reference. This is valid for XML conversion.

INT2FIX(ECONV_UNDEF_HEX_CHARREF)
PARTIAL_INPUT =

Indicates the source may be part of a larger string. See primitive_convert for an example.

INT2FIX(ECONV_PARTIAL_INPUT)
AFTER_OUTPUT =

Stop converting after some output is complete but before all of the input was consumed. See primitive_convert for an example.

INT2FIX(ECONV_AFTER_OUTPUT)
UNIVERSAL_NEWLINE_DECORATOR =

Decorator for converting CRLF and CR to LF

INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECORATOR)
CRLF_NEWLINE_DECORATOR =

Decorator for converting LF to CRLF

INT2FIX(ECONV_CRLF_NEWLINE_DECORATOR)
CR_NEWLINE_DECORATOR =

Decorator for converting LF to CR

INT2FIX(ECONV_CR_NEWLINE_DECORATOR)
XML_TEXT_DECORATOR =

Escape as XML CharData

INT2FIX(ECONV_XML_TEXT_DECORATOR)
XML_ATTR_CONTENT_DECORATOR =

Escape as XML AttValue

INT2FIX(ECONV_XML_ATTR_CONTENT_DECORATOR)
XML_ATTR_QUOTE_DECORATOR =

Escape as XML AttValue

INT2FIX(ECONV_XML_ATTR_QUOTE_DECORATOR)

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#Encoding::Converter.new(source_encoding, destination_encoding) ⇒ Object #Encoding::Converter.new(source_encoding, destination_encoding, opt) ⇒ Object #Encoding::Converter.new(convpath) ⇒ Object

possible options elements:

hash form:
  :invalid => nil            # raise error on invalid byte sequence (default)
  :invalid => :replace       # replace invalid byte sequence
  :undef => nil              # raise error on undefined conversion (default)
  :undef => :replace         # replace undefined conversion
  :replace => string         # replacement string ("?" or "\uFFFD" if not specified)
  :newline => :universal     # decorator for converting CRLF and CR to LF
  :newline => :crlf          # decorator for converting LF to CRLF
  :newline => :cr            # decorator for converting LF to CR
  :universal_newline => true # decorator for converting CRLF and CR to LF
  :crlf_newline => true      # decorator for converting LF to CRLF
  :cr_newline => true        # decorator for converting LF to CR
  :xml => :text              # escape as XML CharData.
  :xml => :attr              # escape as XML AttValue
integer form:
  Encoding::Converter::INVALID_REPLACE
  Encoding::Converter::UNDEF_REPLACE
  Encoding::Converter::UNDEF_HEX_CHARREF
  Encoding::Converter::UNIVERSAL_NEWLINE_DECORATOR
  Encoding::Converter::CRLF_NEWLINE_DECORATOR
  Encoding::Converter::CR_NEWLINE_DECORATOR
  Encoding::Converter::XML_TEXT_DECORATOR
  Encoding::Converter::XML_ATTR_CONTENT_DECORATOR
  Encoding::Converter::XML_ATTR_QUOTE_DECORATOR

Encoding::Converter.new creates an instance of Encoding::Converter.

Source_encoding and destination_encoding should be a string or Encoding object.

opt should be nil, a hash or an integer.

convpath should be an array. convpath may contain

  • two-element arrays which contain encodings or encoding names, or

  • strings representing decorator names.

Encoding::Converter.new optionally takes an option. The option should be a hash or an integer. The option hash can contain :invalid => nil, etc. The option integer should be logical-or of constants such as Encoding::Converter::INVALID_REPLACE, etc.

:invalid => nil

Raise error on invalid byte sequence. This is a default behavior.

:invalid => :replace

Replace invalid byte sequence by replacement string.

:undef => nil

Raise an error if a character in source_encoding is not defined in destination_encoding. This is a default behavior.

:undef => :replace

Replace undefined character in destination_encoding with replacement string.

:replace => string

Specify the replacement string. If not specified, “uFFFD” is used for Unicode encodings and “?” for others.

:universal_newline => true

Convert CRLF and CR to LF.

:crlf_newline => true

Convert LF to CRLF.

:cr_newline => true

Convert LF to CR.

:xml => :text

Escape as XML CharData. This form can be used as an HTML 4.0 #PCDATA.

  • ‘&’ -> ‘&’

  • ‘<’ -> ‘&lt;’

  • ‘>’ -> ‘&gt;’

  • undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;

:xml => :attr

Escape as XML AttValue. The converted result is quoted as “…”. This form can be used as an HTML 4.0 attribute value.

  • ‘&’ -> ‘&amp;’

  • ‘<’ -> ‘&lt;’

  • ‘>’ -> ‘&gt;’

  • ‘“’ -> ‘&quot;’

  • undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;

Examples:

# UTF-16BE to UTF-8
ec = Encoding::Converter.new("UTF-16BE", "UTF-8")

# Usually, decorators such as newline conversion are inserted last.
ec = Encoding::Converter.new("UTF-16BE", "UTF-8", :universal_newline => true)
p ec.convpath #=> [[#<Encoding:UTF-16BE>, #<Encoding:UTF-8>],
              #    "universal_newline"]

# But, if the last encoding is ASCII incompatible,
# decorators are inserted before the last conversion.
ec = Encoding::Converter.new("UTF-8", "UTF-16BE", :crlf_newline => true)
p ec.convpath #=> ["crlf_newline",
              #    [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]

# Conversion path can be specified directly.
ec = Encoding::Converter.new(["universal_newline", ["EUC-JP", "UTF-8"], ["UTF-8", "UTF-16BE"]])
p ec.convpath #=> ["universal_newline",
              #    [#<Encoding:EUC-JP>, #<Encoding:UTF-8>],
              #    [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]



3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
 
# File 'transcode.c', line 3385

static VALUE
econv_init(int argc, VALUE *argv, VALUE self)
{
    VALUE ecopts;
    VALUE snamev, dnamev;
    const char *sname, *dname;
    rb_encoding *senc, *denc;
    rb_econv_t *ec;
    int ecflags;
    VALUE convpath;

    if (rb_check_typeddata(self, &econv_data_type)) {
        rb_raise(rb_eTypeError, "already initialized");
    }

    if (argc == 1 && !NIL_P(convpath = rb_check_array_type(argv[0]))) {
        ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
        ecflags = 0;
        ecopts = Qnil;
    }
    else {
        econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
        ec = rb_econv_open_opts(sname, dname, ecflags, ecopts);
    }

    if (!ec) {
	VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
	RB_GC_GUARD(snamev);
	RB_GC_GUARD(dnamev);
	rb_exc_raise(exc);
    }

    if (!DECORATOR_P(sname, dname)) {
        if (!senc)
            senc = make_dummy_encoding(sname);
        if (!denc)
            denc = make_dummy_encoding(dname);
	RB_GC_GUARD(snamev);
	RB_GC_GUARD(dnamev);
    }

    ec->source_encoding = senc;
    ec->destination_encoding = denc;

    DATA_PTR(self) = ec;

    return self;
}

Class Method Details

.Encoding::Converter.asciicompat_encoding(string) ⇒ Encoding? .Encoding::Converter.asciicompat_encoding(encoding) ⇒ Encoding?

Returns the corresponding ASCII compatible encoding.

Returns nil if the argument is an ASCII compatible encoding.

“corresponding ASCII compatible encoding” is an ASCII compatible encoding which can represents exactly the same characters as the given ASCII incompatible encoding. So, no conversion undefined error occurs when converting between the two encodings.

Encoding::Converter.asciicompat_encoding("ISO-2022-JP") #=> #<Encoding:stateless-ISO-2022-JP>
Encoding::Converter.asciicompat_encoding("UTF-16BE") #=> #<Encoding:UTF-8>
Encoding::Converter.asciicompat_encoding("UTF-8") #=> nil

Overloads:

  • .Encoding::Converter.asciicompat_encoding(string) ⇒ Encoding?

    Returns:

  • .Encoding::Converter.asciicompat_encoding(encoding) ⇒ Encoding?

    Returns:



2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
 
# File 'transcode.c', line 2975

static VALUE
econv_s_asciicompat_encoding(VALUE klass, VALUE arg)
{
    const char *arg_name, *result_name;
    rb_encoding *arg_enc, *result_enc;

    enc_arg(&arg, &arg_name, &arg_enc);

    result_name = rb_econv_asciicompat_encoding(arg_name);

    if (result_name == NULL)
        return Qnil;

    result_enc = make_encoding(result_name);

    return rb_enc_from_encoding(result_enc);
}

.Encoding::Converter.search_convpath(source_encoding, destination_encoding) ⇒ Array .Encoding::Converter.search_convpath(source_encoding, destination_encoding, opt) ⇒ Array

Returns a conversion path.

p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP")
#=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
#    [#<Encoding:UTF-8>, #<Encoding:EUC-JP>]]

p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", universal_newline: true)
or
p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", newline: :universal)
#=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
#    [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
#    "universal_newline"]

p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", universal_newline: true)
or
p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", newline: :universal)
#=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
#    "universal_newline",
#    [#<Encoding:UTF-8>, #<Encoding:UTF-32BE>]]

Overloads:

  • .Encoding::Converter.search_convpath(source_encoding, destination_encoding) ⇒ Array

    Returns:

  • .Encoding::Converter.search_convpath(source_encoding, destination_encoding, opt) ⇒ Array

    Returns:



3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
 
# File 'transcode.c', line 3136

static VALUE
econv_s_search_convpath(int argc, VALUE *argv, VALUE klass)
{
    VALUE snamev, dnamev;
    const char *sname, *dname;
    rb_encoding *senc, *denc;
    int ecflags;
    VALUE ecopts;
    VALUE convpath;

    econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);

    convpath = Qnil;
    transcode_search_path(sname, dname, search_convpath_i, &convpath);

    if (NIL_P(convpath)) {
        VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
        RB_GC_GUARD(snamev);
        RB_GC_GUARD(dnamev);
        rb_exc_raise(exc);
    }

    if (decorate_convpath(convpath, ecflags) == -1) {
	VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
	RB_GC_GUARD(snamev);
	RB_GC_GUARD(dnamev);
	rb_exc_raise(exc);
    }

    return convpath;
}

Instance Method Details

#==(other) ⇒ Boolean

Returns:

  • (Boolean) 


3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
 
# File 'transcode.c', line 3552

static VALUE
econv_equal(VALUE self, VALUE other)
{
    rb_econv_t *ec1 = check_econv(self);
    rb_econv_t *ec2;
    int i;

    if (!rb_typeddata_is_kind_of(other, &econv_data_type)) {
	return Qnil;
    }
    ec2 = DATA_PTR(other);
    if (!ec2) return Qfalse;
    if (ec1->source_encoding_name != ec2->source_encoding_name &&
	strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
	return Qfalse;
    if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
	strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
	return Qfalse;
    if (ec1->flags != ec2->flags) return Qfalse;
    if (ec1->replacement_enc != ec2->replacement_enc &&
	strcmp(ec1->replacement_enc, ec2->replacement_enc))
	return Qfalse;
    if (ec1->replacement_len != ec2->replacement_len) return Qfalse;
    if (ec1->replacement_str != ec2->replacement_str &&
	memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
	return Qfalse;

    if (ec1->num_trans != ec2->num_trans) return Qfalse;
    for (i = 0; i < ec1->num_trans; i++) {
        if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
	    return Qfalse;
    }
    return Qtrue;
}

#convert(source_string) ⇒ Object

Convert source_string and return destination_string.

source_string is assumed as a part of source. i.e. :partial_input=>true is specified internally. finish method should be used last.

ec = Encoding::Converter.new("utf-8", "euc-jp")
puts ec.convert("\u3042").dump     #=> "\xA4\xA2"
puts ec.finish.dump                #=> ""

ec = Encoding::Converter.new("euc-jp", "utf-8")
puts ec.convert("\xA4").dump       #=> ""
puts ec.convert("\xA2").dump       #=> "\xE3\x81\x82"
puts ec.finish.dump                #=> ""

ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
puts ec.convert("\xE3").dump       #=> "".force_encoding("ISO-2022-JP")
puts ec.convert("\x81").dump       #=> "".force_encoding("ISO-2022-JP")
puts ec.convert("\x82").dump       #=> "\e$B$\"".force_encoding("ISO-2022-JP")
puts ec.finish.dump                #=> "\e(B".force_encoding("ISO-2022-JP")

If a conversion error occur, Encoding::UndefinedConversionError or Encoding::InvalidByteSequenceError is raised. Encoding::Converter#convert doesn’t supply methods to recover or restart from these exceptions. When you want to handle these conversion errors, use Encoding::Converter#primitive_convert.



3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
 
# File 'transcode.c', line 3841

static VALUE
econv_convert(VALUE self, VALUE source_string)
{
    VALUE ret, dst;
    VALUE av[5];
    int ac;
    rb_econv_t *ec = check_econv(self);

    StringValue(source_string);

    dst = rb_str_new(NULL, 0);

    av[0] = rb_str_dup(source_string);
    av[1] = dst;
    av[2] = Qnil;
    av[3] = Qnil;
    av[4] = INT2NUM(ECONV_PARTIAL_INPUT);
    ac = 5;

    ret = econv_primitive_convert(ac, av, self);

    if (ret == sym_invalid_byte_sequence ||
        ret == sym_undefined_conversion ||
        ret == sym_incomplete_input) {
        VALUE exc = make_econv_exception(ec);
        rb_exc_raise(exc);
    }

    if (ret == sym_finished) {
        rb_raise(rb_eArgError, "converter already finished");
    }

    if (ret != sym_source_buffer_empty) {
        rb_bug("unexpected result of econv_primitive_convert");
    }

    return dst;
}

#convpath ⇒ Array

Returns the conversion path of ec.

The result is an array of conversions.

ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP", crlf_newline: true)
p ec.convpath
#=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
#    [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
#    "crlf_newline"]

Each element of the array is a pair of encodings or a string. A pair means an encoding conversion. A string means a decorator.

In the above example, [#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>] means a converter from ISO-8859-1 to UTF-8. “crlf_newline” means newline converter from LF to CRLF.

Returns:



3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
 
# File 'transcode.c', line 3528

static VALUE
econv_convpath(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    VALUE result;
    int i;

    result = rb_ary_new();
    for (i = 0; i < ec->num_trans; i++) {
        const rb_transcoder *tr = ec->elems[i].tc->transcoder;
        VALUE v;
        if (DECORATOR_P(tr->src_encoding, tr->dst_encoding))
            v = rb_str_new_cstr(tr->dst_encoding);
        else
            v = rb_assoc_new(make_encobj(tr->src_encoding), make_encobj(tr->dst_encoding));
        rb_ary_push(result, v);
    }
    return result;
}

#destination_encoding ⇒ Encoding

Returns the destination encoding as an Encoding object.

Returns:



3497
3498
3499
3500
3501
3502
3503
3504
 
# File 'transcode.c', line 3497

static VALUE
econv_destination_encoding(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    if (!ec->destination_encoding)
        return Qnil;
    return rb_enc_from_encoding(ec->destination_encoding);
}

#finish ⇒ String

Finishes the converter. It returns the last part of the converted string.

ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
p ec.convert("\u3042")     #=> "\e$B$\""
p ec.finish                #=> "\e(B"

Returns:



3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
 
# File 'transcode.c', line 3891

static VALUE
econv_finish(VALUE self)
{
    VALUE ret, dst;
    VALUE av[5];
    int ac;
    rb_econv_t *ec = check_econv(self);

    dst = rb_str_new(NULL, 0);

    av[0] = Qnil;
    av[1] = dst;
    av[2] = Qnil;
    av[3] = Qnil;
    av[4] = INT2FIX(0);
    ac = 5;

    ret = econv_primitive_convert(ac, av, self);

    if (ret == sym_invalid_byte_sequence ||
        ret == sym_undefined_conversion ||
        ret == sym_incomplete_input) {
        VALUE exc = make_econv_exception(ec);
        rb_exc_raise(exc);
    }

    if (ret != sym_finished) {
        rb_bug("unexpected result of econv_primitive_convert");
    }

    return dst;
}

#insert_output(string) ⇒ nil

Inserts string into the encoding converter. The string will be converted to the destination encoding and output on later conversions.

If the destination encoding is stateful, string is converted according to the state and the state is updated.

This method should be used only when a conversion error occurs.

ec = Encoding::Converter.new("utf-8", "iso-8859-1")
src = "HIRAGANA LETTER A is \u{3042}."
dst = ""
p ec.primitive_convert(src, dst)    #=> :undefined_conversion
puts "[#{dst.dump}, #{src.dump}]"   #=> ["HIRAGANA LETTER A is ", "."]
ec.insert_output("<err>")
p ec.primitive_convert(src, dst)    #=> :finished
puts "[#{dst.dump}, #{src.dump}]"   #=> ["HIRAGANA LETTER A is <err>.", ""]

ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
src = "\u{306F 3041 3068 2661 3002}" # U+2661 is not representable in iso-2022-jp
dst = ""
p ec.primitive_convert(src, dst)    #=> :undefined_conversion
puts "[#{dst.dump}, #{src.dump}]"   #=> ["\e$B$O$!$H".force_encoding("ISO-2022-JP"), "\xE3\x80\x82"]
ec.insert_output "?"                # state change required to output "?".
p ec.primitive_convert(src, dst)    #=> :finished
puts "[#{dst.dump}, #{src.dump}]"   #=> ["\e$B$O$!$H\e(B?\e$B!#\e(B".force_encoding("ISO-2022-JP"), ""]

Returns:

  • (nil) 


4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
 
# File 'transcode.c', line 4057

static VALUE
econv_insert_output(VALUE self, VALUE string)
{
    const char *insert_enc;

    int ret;

    rb_econv_t *ec = check_econv(self);

    StringValue(string);
    insert_enc = rb_econv_encoding_to_insert_output(ec);
    string = rb_str_encode(string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0, Qnil);

    ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc);
    if (ret == -1) {
	rb_raise(rb_eArgError, "too big string");
    }

    return Qnil;
}

#inspect ⇒ String

Returns a printable version of ec

ec = Encoding::Converter.new("iso-8859-1", "utf-8")
puts ec.inspect    #=> #<Encoding::Converter: ISO-8859-1 to UTF-8>

Returns:



3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
 
# File 'transcode.c', line 3444

static VALUE
econv_inspect(VALUE self)
{
    const char *cname = rb_obj_classname(self);
    rb_econv_t *ec;

    TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
    if (!ec)
        return rb_sprintf("#<%s: uninitialized>", cname);
    else {
        const char *sname = ec->source_encoding_name;
        const char *dname = ec->destination_encoding_name;
        VALUE str;
        str = rb_sprintf("#<%s: ", cname);
        econv_description(sname, dname, ec->flags, str);
        rb_str_cat2(str, ">");
        return str;
    }
}

#last_error ⇒ Exception?

Returns an exception object for the last conversion. Returns nil if the last conversion did not produce an error.

“error” means that Encoding::InvalidByteSequenceError and Encoding::UndefinedConversionError for Encoding::Converter#convert and :invalid_byte_sequence, :incomplete_input and :undefined_conversion for Encoding::Converter#primitive_convert.

ec = Encoding::Converter.new("utf-8", "iso-8859-1")
p ec.primitive_convert(src="\xf1abcd", dst="")       #=> :invalid_byte_sequence
p ec.last_error      #=> #<Encoding::InvalidByteSequenceError: "\xF1" followed by "a" on UTF-8>
p ec.primitive_convert(src, dst, nil, 1)             #=> :destination_buffer_full
p ec.last_error      #=> nil

Returns:



4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
 
# File 'transcode.c', line 4150

static VALUE
econv_last_error(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    VALUE exc;

    exc = make_econv_exception(ec);
    if (NIL_P(exc))
        return Qnil;
    return exc;
}

#primitive_convert(source_buffer, destination_buffer) ⇒ Object #primitive_convert(source_buffer, destination_buffer, destination_byteoffset) ⇒ Object #primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize) ⇒ Object #primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize, opt) ⇒ Object

possible opt elements:

hash form:
  :partial_input => true           # source buffer may be part of larger source
  :after_output => true            # stop conversion after output before input
integer form:
  Encoding::Converter::PARTIAL_INPUT
  Encoding::Converter::AFTER_OUTPUT

possible results:

:invalid_byte_sequence
:incomplete_input
:undefined_conversion
:after_output
:destination_buffer_full
:source_buffer_empty
:finished

primitive_convert converts source_buffer into destination_buffer.

source_buffer should be a string or nil. nil means an empty string.

destination_buffer should be a string.

destination_byteoffset should be an integer or nil. nil means the end of destination_buffer. If it is omitted, nil is assumed.

destination_bytesize should be an integer or nil. nil means unlimited. If it is omitted, nil is assumed.

opt should be nil, a hash or an integer. nil means no flags. If it is omitted, nil is assumed.

primitive_convert converts the content of source_buffer from beginning and store the result into destination_buffer.

destination_byteoffset and destination_bytesize specify the region which the converted result is stored. destination_byteoffset specifies the start position in destination_buffer in bytes. If destination_byteoffset is nil, destination_buffer.bytesize is used for appending the result. destination_bytesize specifies maximum number of bytes. If destination_bytesize is nil, destination size is unlimited. After conversion, destination_buffer is resized to destination_byteoffset + actually produced number of bytes. Also destination_buffer’s encoding is set to destination_encoding.

primitive_convert drops the converted part of source_buffer. the dropped part is converted in destination_buffer or buffered in Encoding::Converter object.

primitive_convert stops conversion when one of following condition met.

  • invalid byte sequence found in source buffer (:invalid_byte_sequence) primitive_errinfo and last_error methods returns the detail of the error.

  • unexpected end of source buffer (:incomplete_input) this occur only when :partial_input is not specified. primitive_errinfo and last_error methods returns the detail of the error.

  • character not representable in output encoding (:undefined_conversion) primitive_errinfo and last_error methods returns the detail of the error.

  • after some output is generated, before input is done (:after_output) this occur only when :after_output is specified.

  • destination buffer is full (:destination_buffer_full) this occur only when destination_bytesize is non-nil.

  • source buffer is empty (:source_buffer_empty) this occur only when :partial_input is specified.

  • conversion is finished (:finished)

example:

ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
ret = ec.primitive_convert(src="pi", dst="", nil, 100)
p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"]

ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
ret = ec.primitive_convert(src="pi", dst="", nil, 1)
p [ret, src, dst] #=> [:destination_buffer_full, "i", "\x00"]
ret = ec.primitive_convert(src, dst="", nil, 1)
p [ret, src, dst] #=> [:destination_buffer_full, "", "p"]
ret = ec.primitive_convert(src, dst="", nil, 1)
p [ret, src, dst] #=> [:destination_buffer_full, "", "\x00"]
ret = ec.primitive_convert(src, dst="", nil, 1)
p [ret, src, dst] #=> [:finished, "", "i"]



3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
 
# File 'transcode.c', line 3696

static VALUE
econv_primitive_convert(int argc, VALUE *argv, VALUE self)
{
    VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
    rb_econv_t *ec = check_econv(self);
    rb_econv_result_t res;
    const unsigned char *ip, *is;
    unsigned char *op, *os;
    long output_byteoffset, output_bytesize;
    unsigned long output_byteend;
    int flags;

    argc = rb_scan_args(argc, argv, "23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);

    if (NIL_P(output_byteoffset_v))
        output_byteoffset = 0; /* dummy */
    else
        output_byteoffset = NUM2LONG(output_byteoffset_v);

    if (NIL_P(output_bytesize_v))
        output_bytesize = 0; /* dummy */
    else
        output_bytesize = NUM2LONG(output_bytesize_v);

    if (!NIL_P(flags_v)) {
	if (!NIL_P(opt)) {
	    rb_error_arity(argc + 1, 2, 5);
	}
	flags = NUM2INT(rb_to_int(flags_v));
    }
    else if (!NIL_P(opt)) {
        VALUE v;
        flags = 0;
        v = rb_hash_aref(opt, sym_partial_input);
        if (RTEST(v))
            flags |= ECONV_PARTIAL_INPUT;
        v = rb_hash_aref(opt, sym_after_output);
        if (RTEST(v))
            flags |= ECONV_AFTER_OUTPUT;
    }
    else {
        flags = 0;
    }

    StringValue(output);
    if (!NIL_P(input))
        StringValue(input);
    rb_str_modify(output);

    if (NIL_P(output_bytesize_v)) {
        output_bytesize = RSTRING_EMBED_LEN_MAX;
        if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input))
            output_bytesize = RSTRING_LEN(input);
    }

  retry:

    if (NIL_P(output_byteoffset_v))
        output_byteoffset = RSTRING_LEN(output);

    if (output_byteoffset < 0)
        rb_raise(rb_eArgError, "negative output_byteoffset");

    if (RSTRING_LEN(output) < output_byteoffset)
        rb_raise(rb_eArgError, "output_byteoffset too big");

    if (output_bytesize < 0)
        rb_raise(rb_eArgError, "negative output_bytesize");

    output_byteend = (unsigned long)output_byteoffset +
                     (unsigned long)output_bytesize;

    if (output_byteend < (unsigned long)output_byteoffset ||
        LONG_MAX < output_byteend)
        rb_raise(rb_eArgError, "output_byteoffset+output_bytesize too big");

    if (rb_str_capacity(output) < output_byteend)
        rb_str_resize(output, output_byteend);

    if (NIL_P(input)) {
        ip = is = NULL;
    }
    else {
        ip = (const unsigned char *)RSTRING_PTR(input);
        is = ip + RSTRING_LEN(input);
    }

    op = (unsigned char *)RSTRING_PTR(output) + output_byteoffset;
    os = op + output_bytesize;

    res = rb_econv_convert(ec, &ip, is, &op, os, flags);
    rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output));
    if (!NIL_P(input)) {
        rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input));
    }

    if (NIL_P(output_bytesize_v) && res == econv_destination_buffer_full) {
        if (LONG_MAX / 2 < output_bytesize)
            rb_raise(rb_eArgError, "too long conversion result");
        output_bytesize *= 2;
        output_byteoffset_v = Qnil;
        goto retry;
    }

    if (ec->destination_encoding) {
        rb_enc_associate(output, ec->destination_encoding);
    }

    return econv_result_to_symbol(res);
}

#primitive_errinfo ⇒ Array

primitive_errinfo returns important information regarding the last error as a 5-element array:

[result, enc1, enc2, error_bytes, readagain_bytes]

result is the last result of primitive_convert.

Other elements are only meaningful when result is :invalid_byte_sequence, :incomplete_input or :undefined_conversion.

enc1 and enc2 indicate a conversion step as a pair of strings. For example, a converter from EUC-JP to ISO-8859-1 converts a string as follows: EUC-JP -> UTF-8 -> ISO-8859-1. So [enc1, enc2] is either [“EUC-JP”, “UTF-8”] or [“UTF-8”, “ISO-8859-1”].

error_bytes and readagain_bytes indicate the byte sequences which caused the error. error_bytes is discarded portion. readagain_bytes is buffered portion which is read again on next conversion.

Example:

# \xff is invalid as EUC-JP.
ec = Encoding::Converter.new("EUC-JP", "Shift_JIS")
ec.primitive_convert(src="\xff", dst="", nil, 10)
p ec.primitive_errinfo
#=> [:invalid_byte_sequence, "EUC-JP", "UTF-8", "\xFF", ""]

# HIRAGANA LETTER A (\xa4\xa2 in EUC-JP) is not representable in ISO-8859-1.
# Since this error is occur in UTF-8 to ISO-8859-1 conversion,
# error_bytes is HIRAGANA LETTER A in UTF-8 (\xE3\x81\x82).
ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
ec.primitive_convert(src="\xa4\xa2", dst="", nil, 10)
p ec.primitive_errinfo
#=> [:undefined_conversion, "UTF-8", "ISO-8859-1", "\xE3\x81\x82", ""]

# partial character is invalid
ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
ec.primitive_convert(src="\xa4", dst="", nil, 10)
p ec.primitive_errinfo
#=> [:incomplete_input, "EUC-JP", "UTF-8", "\xA4", ""]

# Encoding::Converter::PARTIAL_INPUT prevents invalid errors by
# partial characters.
ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
ec.primitive_convert(src="\xa4", dst="", nil, 10, Encoding::Converter::PARTIAL_INPUT)
p ec.primitive_errinfo
#=> [:source_buffer_empty, nil, nil, nil, nil]

# \xd8\x00\x00@ is invalid as UTF-16BE because
# no low surrogate after high surrogate (\xd8\x00).
# It is detected by 3rd byte (\00) which is part of next character.
# So the high surrogate (\xd8\x00) is discarded and
# the 3rd byte is read again later.
# Since the byte is buffered in ec, it is dropped from src.
ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
ec.primitive_convert(src="\xd8\x00\x00@", dst="", nil, 10)
p ec.primitive_errinfo
#=> [:invalid_byte_sequence, "UTF-16BE", "UTF-8", "\xD8\x00", "\x00"]
p src
#=> "@"

# Similar to UTF-16BE, \x00\xd8@\x00 is invalid as UTF-16LE.
# The problem is detected by 4th byte.
ec = Encoding::Converter.new("UTF-16LE", "UTF-8")
ec.primitive_convert(src="\x00\xd8@\x00", dst="", nil, 10)
p ec.primitive_errinfo
#=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "@\x00"]
p src
#=> ""

Returns:



3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
 
# File 'transcode.c', line 3999

static VALUE
econv_primitive_errinfo(VALUE self)
{
    rb_econv_t *ec = check_econv(self);

    VALUE ary;

    ary = rb_ary_new2(5);

    rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
    rb_ary_store(ary, 4, Qnil);

    if (ec->last_error.source_encoding)
        rb_ary_store(ary, 1, rb_str_new2(ec->last_error.source_encoding));

    if (ec->last_error.destination_encoding)
        rb_ary_store(ary, 2, rb_str_new2(ec->last_error.destination_encoding));

    if (ec->last_error.error_bytes_start) {
        rb_ary_store(ary, 3, rb_str_new((const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
        rb_ary_store(ary, 4, rb_str_new((const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
    }

    return ary;
}

#putback ⇒ String #putback(max_numbytes) ⇒ String

Put back the bytes which will be converted.

The bytes are caused by invalid_byte_sequence error. When invalid_byte_sequence error, some bytes are discarded and some bytes are buffered to be converted later. The latter bytes can be put back. It can be observed by Encoding::InvalidByteSequenceError#readagain_bytes and Encoding::Converter#primitive_errinfo.

ec = Encoding::Converter.new("utf-16le", "iso-8859-1")
src = "\x00\xd8\x61\x00"
dst = ""
p ec.primitive_convert(src, dst)   #=> :invalid_byte_sequence
p ec.primitive_errinfo     #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "a\x00"]
p ec.putback               #=> "a\x00"
p ec.putback               #=> ""          # no more bytes to put back

Overloads:



4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
 
# File 'transcode.c', line 4102

static VALUE
econv_putback(int argc, VALUE *argv, VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    int n;
    int putbackable;
    VALUE str, max;

    if (!rb_check_arity(argc, 0, 1) || NIL_P(max = argv[0])) {
        n = rb_econv_putbackable(ec);
    }
    else {
        n = NUM2INT(max);
        putbackable = rb_econv_putbackable(ec);
        if (putbackable < n)
            n = putbackable;
    }

    str = rb_str_new(NULL, n);
    rb_econv_putback(ec, (unsigned char *)RSTRING_PTR(str), n);

    if (ec->source_encoding) {
        rb_enc_associate(str, ec->source_encoding);
    }

    return str;
}

#replacement ⇒ String

Returns the replacement string.

ec = Encoding::Converter.new("euc-jp", "us-ascii")
p ec.replacement    #=> "?"

ec = Encoding::Converter.new("euc-jp", "utf-8")
p ec.replacement    #=> "\uFFFD"

Returns:



4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
 
# File 'transcode.c', line 4174

static VALUE
econv_get_replacement(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    int ret;
    rb_encoding *enc;

    ret = make_replacement(ec);
    if (ret == -1) {
        rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
    }

    enc = rb_enc_find(ec->replacement_enc);
    return rb_enc_str_new((const char *)ec->replacement_str, (long)ec->replacement_len, enc);
}

#replacement=(string) ⇒ Object

Sets the replacement string.

ec = Encoding::Converter.new("utf-8", "us-ascii", :undef => :replace)
ec.replacement = "<undef>"
p ec.convert("a \u3042 b")      #=> "a <undef> b"



4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
 
# File 'transcode.c', line 4200

static VALUE
econv_set_replacement(VALUE self, VALUE arg)
{
    rb_econv_t *ec = check_econv(self);
    VALUE string = arg;
    int ret;
    rb_encoding *enc;

    StringValue(string);
    enc = rb_enc_get(string);

    ret = rb_econv_set_replacement(ec,
            (const unsigned char *)RSTRING_PTR(string),
            RSTRING_LEN(string),
            rb_enc_name(enc));

    if (ret == -1) {
        /* xxx: rb_eInvalidByteSequenceError? */
        rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
    }

    return arg;
}

#source_encoding ⇒ Encoding

Returns the source encoding as an Encoding object.

Returns:



3482
3483
3484
3485
3486
3487
3488
3489
 
# File 'transcode.c', line 3482

static VALUE
econv_source_encoding(VALUE self)
{
    rb_econv_t *ec = check_econv(self);
    if (!ec->source_encoding)
        return Qnil;
    return rb_enc_from_encoding(ec->source_encoding);
}