kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

io.c: conversion from bom encoding

Browse source 
* io.c (rb_io_ext_int_to_encs, parse_mode_enc): bom-prefixed name is
  not a real encoding name, just a fallback.  so the proper conversion
  should take place even if if the internal encoding is equal to the
  bom-prefixed name, unless actual encoding is equal to the internal
  encoding.  [ruby-core:54563] [Bug #8323]
* io.c (io_set_encoding_by_bom): reset extenal encoding if no BOM
  found.  [ruby-core:54569]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@40462 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
nobu 2013-04-25 07:11:20 +00:00
parent 8b29525dad
commit 91fb5bc889
3 changed files with 57 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
ChangeLog
View file

@ -1,3 +1,14 @@
Thu Apr 25 16:11:06 2013 Nobuyoshi Nakada <nobu@ruby-lang.org>
* io.c (rb_io_ext_int_to_encs, parse_mode_enc): bom-prefixed name is
not a real encoding name, just a fallback. so the proper conversion
should take place even if if the internal encoding is equal to the
bom-prefixed name, unless actual encoding is equal to the internal
encoding. [ruby-core:54563] [Bug #8323]
* io.c (io_set_encoding_by_bom): reset extenal encoding if no BOM
found. [ruby-core:54569]
Thu Apr 25 14:35:01 2013 NARUSE, Yui <naruse@ruby-lang.org>
* ext/openssl/ossl_bn.c (ossl_bn_initialize): allow Fixnum and Bignum.

40
io.c
View file

@ -4860,7 +4860,7 @@ rb_io_oflags_modestr(int oflags)
* Qnil => no encoding specified (internal only)
*/
static void
rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2)
rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2, int fmode)
{
int default_ext = 0;
@ -4871,7 +4871,8 @@ rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc,
if (intern == NULL && ext != rb_ascii8bit_encoding())
/* If external is ASCII-8BIT, no default transcoding */
intern = rb_default_internal_encoding();
if (intern == NULL || intern == (rb_encoding *)Qnil || intern == ext) {
if (intern == NULL || intern == (rb_encoding *)Qnil ||
(!(fmode & FMODE_SETENC_BY_BOM) && (intern == ext))) {
/* No internal encoding => use external + no transcoding */
*enc = (default_ext && intern != ext) ? NULL : ext;
*enc2 = NULL;
@ -4894,6 +4895,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
const char *p;
char encname[ENCODING_MAXNAMELEN+1];
int idx, idx2;
int fmode = fmode_p ? *fmode_p : 0;
rb_encoding *ext_enc, *int_enc;
/* parse estr as "enc" or "enc2:enc" or "enc:-" */
@ -4905,7 +4907,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
idx = -1;
else {
if (io_encname_bom_p(estr, len)) {
if (fmode_p) *fmode_p |= FMODE_SETENC_BY_BOM;
fmode |= FMODE_SETENC_BY_BOM;
estr += 4;
len -= 4;
}
@ -4918,7 +4920,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
else {
long len = strlen(estr);
if (io_encname_bom_p(estr, len)) {
if (fmode_p) *fmode_p |= FMODE_SETENC_BY_BOM;
fmode |= FMODE_SETENC_BY_BOM;
estr += 4;
len -= 4;
memcpy(encname, estr, len);
@ -4927,6 +4929,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
}
idx = rb_enc_find_index(estr);
}
if (fmode_p) *fmode_p = fmode;
if (idx >= 0)
ext_enc = rb_enc_from_index(idx);
@ -4946,7 +4949,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
idx2 = rb_enc_find_index(p);
if (idx2 < 0)
unsupported_encoding(p);
else if (idx2 == idx) {
else if (!(fmode & FMODE_SETENC_BY_BOM) && (idx2 == idx)) {
int_enc = (rb_encoding *)Qnil;
}
else
@ -4954,7 +4957,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
}
}
rb_io_ext_int_to_encs(ext_enc, int_enc, enc_p, enc2_p);
rb_io_ext_int_to_encs(ext_enc, int_enc, enc_p, enc2_p, fmode);
}
int
@ -5015,12 +5018,12 @@ rb_io_extract_encoding_option(VALUE opt, rb_encoding **enc_p, rb_encoding **enc2
parse_mode_enc(StringValueCStr(tmp), enc_p, enc2_p, fmode_p);
}
else {
rb_io_ext_int_to_encs(rb_to_encoding(encoding), NULL, enc_p, enc2_p);
rb_io_ext_int_to_encs(rb_to_encoding(encoding), NULL, enc_p, enc2_p, 0);
}
}
else if (extenc != Qundef || intenc != Qundef) {
extracted = 1;
rb_io_ext_int_to_encs(extencoding, intencoding, enc_p, enc2_p);
rb_io_ext_int_to_encs(extencoding, intencoding, enc_p, enc2_p, 0);
}
return extracted;
}
@ -5095,7 +5098,7 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
vmode = *vmode_p;
/* Set to defaults */
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2);
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2, 0);
vmode_handle:
if (NIL_P(vmode)) {
@ -5123,7 +5126,7 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
rb_encoding *e;
e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL;
rb_io_ext_int_to_encs(e, NULL, &enc, &enc2);
rb_io_ext_int_to_encs(e, NULL, &enc, &enc2, fmode);
}
}
@ -5147,7 +5150,7 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
oflags |= O_BINARY;
#endif
if (!has_enc)
rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2);
rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2, fmode);
}
#if DEFAULT_TEXTMODE
else if (NIL_P(vmode)) {
@ -5370,13 +5373,16 @@ static void
io_set_encoding_by_bom(VALUE io)
{
int idx = io_strip_bom(io);
rb_io_t *fptr;
GetOpenFile(io, fptr);
if (idx) {
rb_io_t *fptr;
GetOpenFile(io, fptr);
io_encoding_set(fptr, rb_enc_from_encoding(rb_enc_from_index(idx)),
rb_io_internal_encoding(io), Qnil);
}
else {
fptr->encs.enc2 = NULL;
}
}
static VALUE
@ -5386,7 +5392,7 @@ rb_file_open_generic(VALUE io, VALUE filename, int oflags, int fmode, convconfig
convconfig_t cc;
if (!convconfig) {
/* Set to default encodings */
rb_io_ext_int_to_encs(NULL, NULL, &cc.enc, &cc.enc2);
rb_io_ext_int_to_encs(NULL, NULL, &cc.enc, &cc.enc2, fmode);
cc.ecflags = 0;
cc.ecopts = Qnil;
convconfig = &cc;
@ -5420,7 +5426,7 @@ rb_file_open_internal(VALUE io, VALUE filename, const char *modestr)
/* Set to default encodings */
e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL;
rb_io_ext_int_to_encs(e, NULL, &convconfig.enc, &convconfig.enc2);
rb_io_ext_int_to_encs(e, NULL, &convconfig.enc, &convconfig.enc2, fmode);
convconfig.ecflags = 0;
convconfig.ecopts = Qnil;
}
@ -9078,7 +9084,7 @@ io_encoding_set(rb_io_t *fptr, VALUE v1, VALUE v2, VALUE opt)
else {
if (NIL_P(v1)) {
/* Set to default encodings */
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2);
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2, 0);
SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
ecopts = Qnil;
}
@ -9090,7 +9096,7 @@ io_encoding_set(rb_io_t *fptr, VALUE v1, VALUE v2, VALUE opt)
ecflags = rb_econv_prepare_options(opt, &ecopts, ecflags);
}
else {
rb_io_ext_int_to_encs(find_encoding(v1), NULL, &enc, &enc2);
rb_io_ext_int_to_encs(find_encoding(v1), NULL, &enc, &enc2, 0);
SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
ecopts = Qnil;
}

24
test/ruby/test_io_m17n.rb
View file

@ -1997,6 +1997,7 @@ EOT
def test_strip_bom
with_tmpdir {
text = "\uFEFFa"
stripped = "a"
%w/UTF-8 UTF-16BE UTF-16LE UTF-32BE UTF-32LE/.each do |name|
path = '%s-bom.txt' % name
content = text.encode(name)
@ -2004,11 +2005,32 @@ EOT
result = File.read(path, mode: 'rb:BOM|UTF-8')
assert_equal(content[1].force_encoding("ascii-8bit"),
result.force_encoding("ascii-8bit"))
result = File.read(path, mode: 'rb:BOM|UTF-8:UTF-8')
assert_equal(Encoding::UTF_8, result.encoding)
assert_equal(stripped, result)
end
bug3407 = '[ruby-core:30641]'
result = File.read('UTF-8-bom.txt', encoding: 'BOM|UTF-8')
path = 'UTF-8-bom.txt'
result = File.read(path, encoding: 'BOM|UTF-8')
assert_equal("a", result.force_encoding("ascii-8bit"), bug3407)
bug8323 = '[ruby-core:54563] [Bug #8323]'
expected = "a\xff".force_encoding("utf-8")
open(path, 'ab') {|f| f.write("\xff")}
result = File.read(path, encoding: 'BOM|UTF-8')
assert_not_predicate(result, :valid_encoding?, bug8323)
assert_equal(expected, result, bug8323)
result = File.read(path, encoding: 'BOM|UTF-8:UTF-8')
assert_not_predicate(result, :valid_encoding?, bug8323)
assert_equal(expected, result, bug8323)
path = 'ascii.txt'
generate_file(path, stripped)
result = File.read(path, encoding: 'BOM|UTF-8')
assert_equal(stripped, result, bug8323)
result = File.read(path, encoding: 'BOM|UTF-8:UTF-8')
assert_equal(stripped, result, bug8323)
}
end