kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

io.c: conversion from bom encoding

Browse source 
* io.c (rb_io_ext_int_to_encs, parse_mode_enc): bom-prefixed name is
  not a real encoding name, just a fallback.  so the proper conversion
  should take place even if if the internal encoding is equal to the
  bom-prefixed name, unless actual encoding is equal to the internal
  encoding.  [ruby-core:54563] [Bug #8323]
* io.c (io_set_encoding_by_bom): reset extenal encoding if no BOM
  found.  [ruby-core:54569]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@40462 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
nobu 2013-04-25 07:11:20 +00:00
parent 8b29525dad
commit 91fb5bc889
3 changed files with 57 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
ChangeLog
View file

@ -1,3 +1,14 @@
Thu Apr 25 16:11:06 2013 Nobuyoshi Nakada <nobu@ruby-lang.org>
* io.c (rb_io_ext_int_to_encs, parse_mode_enc): bom-prefixed name is
not a real encoding name, just a fallback. so the proper conversion
should take place even if if the internal encoding is equal to the
bom-prefixed name, unless actual encoding is equal to the internal
encoding. [ruby-core:54563] [Bug #8323]
* io.c (io_set_encoding_by_bom): reset extenal encoding if no BOM
found. [ruby-core:54569]
Thu Apr 25 14:35:01 2013 NARUSE, Yui <naruse@ruby-lang.org> Thu Apr 25 14:35:01 2013 NARUSE, Yui <naruse@ruby-lang.org>
* ext/openssl/ossl_bn.c (ossl_bn_initialize): allow Fixnum and Bignum. * ext/openssl/ossl_bn.c (ossl_bn_initialize): allow Fixnum and Bignum.

40
io.c
View file

@ -4860,7 +4860,7 @@ rb_io_oflags_modestr(int oflags)
* Qnil => no encoding specified (internal only) * Qnil => no encoding specified (internal only)
*/ */
static void static void
rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2) rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2, int fmode)
{ {
int default_ext = 0; int default_ext = 0;
@ -4871,7 +4871,8 @@ rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc,
if (intern == NULL && ext != rb_ascii8bit_encoding()) if (intern == NULL && ext != rb_ascii8bit_encoding())
/* If external is ASCII-8BIT, no default transcoding */ /* If external is ASCII-8BIT, no default transcoding */
intern = rb_default_internal_encoding(); intern = rb_default_internal_encoding();
if (intern == NULL || intern == (rb_encoding *)Qnil || intern == ext) { if (intern == NULL || intern == (rb_encoding *)Qnil ||
(!(fmode & FMODE_SETENC_BY_BOM) && (intern == ext))) {
/* No internal encoding => use external + no transcoding */ /* No internal encoding => use external + no transcoding */
*enc = (default_ext && intern != ext) ? NULL : ext; *enc = (default_ext && intern != ext) ? NULL : ext;
*enc2 = NULL; *enc2 = NULL;
@ -4894,6 +4895,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
const char *p; const char *p;
char encname[ENCODING_MAXNAMELEN+1]; char encname[ENCODING_MAXNAMELEN+1];
int idx, idx2; int idx, idx2;
int fmode = fmode_p ? *fmode_p : 0;
rb_encoding *ext_enc, *int_enc; rb_encoding *ext_enc, *int_enc;
/* parse estr as "enc" or "enc2:enc" or "enc:-" */ /* parse estr as "enc" or "enc2:enc" or "enc:-" */
@ -4905,7 +4907,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
idx = -1; idx = -1;
else { else {
if (io_encname_bom_p(estr, len)) { if (io_encname_bom_p(estr, len)) {
if (fmode_p) *fmode_p |= FMODE_SETENC_BY_BOM; fmode |= FMODE_SETENC_BY_BOM;
estr += 4; estr += 4;
len -= 4; len -= 4;
} }
@ -4918,7 +4920,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
else { else {
long len = strlen(estr); long len = strlen(estr);
if (io_encname_bom_p(estr, len)) { if (io_encname_bom_p(estr, len)) {
if (fmode_p) *fmode_p |= FMODE_SETENC_BY_BOM; fmode |= FMODE_SETENC_BY_BOM;
estr += 4; estr += 4;
len -= 4; len -= 4;
memcpy(encname, estr, len); memcpy(encname, estr, len);
@ -4927,6 +4929,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
} }
idx = rb_enc_find_index(estr); idx = rb_enc_find_index(estr);
} }
if (fmode_p) *fmode_p = fmode;
if (idx >= 0) if (idx >= 0)
ext_enc = rb_enc_from_index(idx); ext_enc = rb_enc_from_index(idx);
@ -4946,7 +4949,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
idx2 = rb_enc_find_index(p); idx2 = rb_enc_find_index(p);
if (idx2 < 0) if (idx2 < 0)
unsupported_encoding(p); unsupported_encoding(p);
else if (idx2 == idx) { else if (!(fmode & FMODE_SETENC_BY_BOM) && (idx2 == idx)) {
int_enc = (rb_encoding *)Qnil; int_enc = (rb_encoding *)Qnil;
} }
else else
@ -4954,7 +4957,7 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
} }
} }
rb_io_ext_int_to_encs(ext_enc, int_enc, enc_p, enc2_p); rb_io_ext_int_to_encs(ext_enc, int_enc, enc_p, enc2_p, fmode);
} }
int int
@ -5015,12 +5018,12 @@ rb_io_extract_encoding_option(VALUE opt, rb_encoding **enc_p, rb_encoding **enc2
parse_mode_enc(StringValueCStr(tmp), enc_p, enc2_p, fmode_p); parse_mode_enc(StringValueCStr(tmp), enc_p, enc2_p, fmode_p);
} }
else { else {
rb_io_ext_int_to_encs(rb_to_encoding(encoding), NULL, enc_p, enc2_p); rb_io_ext_int_to_encs(rb_to_encoding(encoding), NULL, enc_p, enc2_p, 0);
} }
} }
else if (extenc != Qundef || intenc != Qundef) { else if (extenc != Qundef || intenc != Qundef) {
extracted = 1; extracted = 1;
rb_io_ext_int_to_encs(extencoding, intencoding, enc_p, enc2_p); rb_io_ext_int_to_encs(extencoding, intencoding, enc_p, enc2_p, 0);
} }
return extracted; return extracted;
} }
@ -5095,7 +5098,7 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
vmode = *vmode_p; vmode = *vmode_p;
/* Set to defaults */ /* Set to defaults */
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2); rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2, 0);
vmode_handle: vmode_handle:
if (NIL_P(vmode)) { if (NIL_P(vmode)) {
@ -5123,7 +5126,7 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
rb_encoding *e; rb_encoding *e;
e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL; e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL;
rb_io_ext_int_to_encs(e, NULL, &enc, &enc2); rb_io_ext_int_to_encs(e, NULL, &enc, &enc2, fmode);
} }
} }
@ -5147,7 +5150,7 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
oflags |= O_BINARY; oflags |= O_BINARY;
#endif #endif
if (!has_enc) if (!has_enc)
rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2); rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2, fmode);
} }
#if DEFAULT_TEXTMODE #if DEFAULT_TEXTMODE
else if (NIL_P(vmode)) { else if (NIL_P(vmode)) {
@ -5370,13 +5373,16 @@ static void
io_set_encoding_by_bom(VALUE io) io_set_encoding_by_bom(VALUE io)
{ {
int idx = io_strip_bom(io); int idx = io_strip_bom(io);
if (idx) {
rb_io_t *fptr; rb_io_t *fptr;
GetOpenFile(io, fptr); GetOpenFile(io, fptr);
if (idx) {
io_encoding_set(fptr, rb_enc_from_encoding(rb_enc_from_index(idx)), io_encoding_set(fptr, rb_enc_from_encoding(rb_enc_from_index(idx)),
rb_io_internal_encoding(io), Qnil); rb_io_internal_encoding(io), Qnil);
} }
else {
fptr->encs.enc2 = NULL;
}
} }
static VALUE static VALUE
@ -5386,7 +5392,7 @@ rb_file_open_generic(VALUE io, VALUE filename, int oflags, int fmode, convconfig
convconfig_t cc; convconfig_t cc;
if (!convconfig) { if (!convconfig) {
/* Set to default encodings */ /* Set to default encodings */
rb_io_ext_int_to_encs(NULL, NULL, &cc.enc, &cc.enc2); rb_io_ext_int_to_encs(NULL, NULL, &cc.enc, &cc.enc2, fmode);
cc.ecflags = 0; cc.ecflags = 0;
cc.ecopts = Qnil; cc.ecopts = Qnil;
convconfig = &cc; convconfig = &cc;
@ -5420,7 +5426,7 @@ rb_file_open_internal(VALUE io, VALUE filename, const char *modestr)
/* Set to default encodings */ /* Set to default encodings */
e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL; e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL;
rb_io_ext_int_to_encs(e, NULL, &convconfig.enc, &convconfig.enc2); rb_io_ext_int_to_encs(e, NULL, &convconfig.enc, &convconfig.enc2, fmode);
convconfig.ecflags = 0; convconfig.ecflags = 0;
convconfig.ecopts = Qnil; convconfig.ecopts = Qnil;
} }
@ -9078,7 +9084,7 @@ io_encoding_set(rb_io_t *fptr, VALUE v1, VALUE v2, VALUE opt)
else { else {
if (NIL_P(v1)) { if (NIL_P(v1)) {
/* Set to default encodings */ /* Set to default encodings */
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2); rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2, 0);
SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags); SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
ecopts = Qnil; ecopts = Qnil;
} }
@ -9090,7 +9096,7 @@ io_encoding_set(rb_io_t *fptr, VALUE v1, VALUE v2, VALUE opt)
ecflags = rb_econv_prepare_options(opt, &ecopts, ecflags); ecflags = rb_econv_prepare_options(opt, &ecopts, ecflags);
} }
else { else {
rb_io_ext_int_to_encs(find_encoding(v1), NULL, &enc, &enc2); rb_io_ext_int_to_encs(find_encoding(v1), NULL, &enc, &enc2, 0);
SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags); SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
ecopts = Qnil; ecopts = Qnil;
} }

24
test/ruby/test_io_m17n.rb
View file

@ -1997,6 +1997,7 @@ EOT
def test_strip_bom def test_strip_bom
with_tmpdir { with_tmpdir {
text = "\uFEFFa" text = "\uFEFFa"
stripped = "a"
%w/UTF-8 UTF-16BE UTF-16LE UTF-32BE UTF-32LE/.each do |name| %w/UTF-8 UTF-16BE UTF-16LE UTF-32BE UTF-32LE/.each do |name|
path = '%s-bom.txt' % name path = '%s-bom.txt' % name
content = text.encode(name) content = text.encode(name)
@ -2004,11 +2005,32 @@ EOT
result = File.read(path, mode: 'rb:BOM|UTF-8') result = File.read(path, mode: 'rb:BOM|UTF-8')
assert_equal(content[1].force_encoding("ascii-8bit"), assert_equal(content[1].force_encoding("ascii-8bit"),
result.force_encoding("ascii-8bit")) result.force_encoding("ascii-8bit"))
result = File.read(path, mode: 'rb:BOM|UTF-8:UTF-8')
assert_equal(Encoding::UTF_8, result.encoding)
assert_equal(stripped, result)
end end
bug3407 = '[ruby-core:30641]' bug3407 = '[ruby-core:30641]'
result = File.read('UTF-8-bom.txt', encoding: 'BOM|UTF-8') path = 'UTF-8-bom.txt'
result = File.read(path, encoding: 'BOM|UTF-8')
assert_equal("a", result.force_encoding("ascii-8bit"), bug3407) assert_equal("a", result.force_encoding("ascii-8bit"), bug3407)
bug8323 = '[ruby-core:54563] [Bug #8323]'
expected = "a\xff".force_encoding("utf-8")
open(path, 'ab') {|f| f.write("\xff")}
result = File.read(path, encoding: 'BOM|UTF-8')
assert_not_predicate(result, :valid_encoding?, bug8323)
assert_equal(expected, result, bug8323)
result = File.read(path, encoding: 'BOM|UTF-8:UTF-8')
assert_not_predicate(result, :valid_encoding?, bug8323)
assert_equal(expected, result, bug8323)
path = 'ascii.txt'
generate_file(path, stripped)
result = File.read(path, encoding: 'BOM|UTF-8')
assert_equal(stripped, result, bug8323)
result = File.read(path, encoding: 'BOM|UTF-8:UTF-8')
assert_equal(stripped, result, bug8323)
} }
end end