kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

* string.c (str_scrub): add ruby method String#scrub which verify and

Browse source 
fix invalid byte sequence.

* string.c (str_compat_and_valid): check given string is compatible
  and valid with given encoding.

* transcode.c (str_transcode0): If invalid: :replace is specified for
  String#encode, replace invalid byte sequence even if the destination
  encoding equals to the source encoding.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@40390 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
naruse 2013-04-19 17:50:38 +00:00
parent 57ffc79c45
commit 394d5dfa9b
5 changed files with 334 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
ChangeLog
View file

@ -1,3 +1,15 @@
Sat Apr 20 02:33:27 2013 NARUSE, Yui <naruse@ruby-lang.org>
* string.c (str_scrub): add ruby method String#scrub which verify and
fix invalid byte sequence. [ruby-dev:45975] [Feature #6752]
* string.c (str_compat_and_valid): check given string is compatible
and valid with given encoding.
* transcode.c (str_transcode0): If invalid: :replace is specified for
String#encode, replace invalid byte sequence even if the destination
encoding equals to the source encoding.
Fri Apr 19 21:55:40 2013 Kouhei Sutou <kou@cozmixng.org>
* README.EXT.ja (Data_Wrap_Struct): Remove a description about

8
NEWS
View file

@ -26,6 +26,14 @@ with all sufficient information, see the ChangeLog file.
* misc
* Mutex#owned? is no longer experimental.
* String
* New methods:
* added String#scrub to verify and fix invalid byte sequence.
* extended methods:
* If invalid: :replace is specified for String#encode, replace
invalid byte sequence even if the destination encoding equals to
the source encoding.
* pack/unpack (Array/String)
* Q! and q! directives for long long type if platform has the type.

267
string.c
View file

@ -7741,6 +7741,272 @@ rb_str_ellipsize(VALUE str, long len)
return ret;
}
static VALUE
str_compat_and_valid(VALUE str, rb_encoding *enc)
{
int cr;
str = StringValue(str);
cr = rb_enc_str_coderange(str);
if (cr == ENC_CODERANGE_BROKEN) {
rb_raise(rb_eArgError, "replacement must be valid byte sequence '%+"PRIsVALUE"'", str);
}
else if (cr == ENC_CODERANGE_7BIT) {
rb_encoding *e = STR_ENC_GET(str);
if (!rb_enc_asciicompat(enc)) {
rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s",
rb_enc_name(enc), rb_enc_name(e));
}
}
else { /* ENC_CODERANGE_VALID */
rb_encoding *e = STR_ENC_GET(str);
if (enc != e) {
rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s",
rb_enc_name(enc), rb_enc_name(e));
}
}
return str;
}
/*
* call-seq:
* str.scrub -> new_str
* str.scrub(repl) -> new_str
* str.scrub{|bytes|} -> new_str
*
* If the string is invalid byte sequence then replace invalid bytes with given replacement
* character, else returns self.
* If block is given, replace invalid bytes with returned value of the block.
*/
VALUE
rb_str_scrub(int argc, VALUE *argv, VALUE str)
{
int cr = ENC_CODERANGE(str);
rb_encoding *enc;
VALUE repl;
if (cr == ENC_CODERANGE_7BIT || cr == ENC_CODERANGE_VALID)
return rb_str_dup(str);
enc = STR_ENC_GET(str);
rb_scan_args(argc, argv, "01", &repl);
if (argc != 0) {
repl = str_compat_and_valid(repl, enc);
}
if (rb_enc_dummy_p(enc)) {
return rb_str_dup(str);
}
if (rb_enc_asciicompat(enc)) {
const char *p = RSTRING_PTR(str);
const char *e = RSTRING_END(str);
const char *p1 = p;
const char *rep;
long replen;
int rep7bit_p;
VALUE buf = rb_str_buf_new(RSTRING_LEN(str));
if (rb_block_given_p()) {
rep = NULL;
}
else if (!NIL_P(repl)) {
rep = RSTRING_PTR(repl);
replen = RSTRING_LEN(repl);
rep7bit_p = (ENC_CODERANGE(repl) == ENC_CODERANGE_7BIT);
}
else if (enc == rb_utf8_encoding()) {
rep = "\xEF\xBF\xBD";
replen = strlen(rep);
rep7bit_p = FALSE;
}
else {
rep = "?";
replen = strlen(rep);
rep7bit_p = TRUE;
}
cr = ENC_CODERANGE_7BIT;
p = search_nonascii(p, e);
if (!p) {
p = e;
}
while (p < e) {
int ret = rb_enc_precise_mbclen(p, e, enc);
if (MBCLEN_NEEDMORE_P(ret)) {
break;
}
else if (MBCLEN_CHARFOUND_P(ret)) {
cr = ENC_CODERANGE_VALID;
p += MBCLEN_CHARFOUND_LEN(ret);
}
else if (MBCLEN_INVALID_P(ret)) {
/*
* p1~p: valid ascii/multibyte chars
* p ~e: invalid bytes + unknown bytes
*/
long clen = rb_enc_mbmaxlen(enc);
if (p > p1) {
rb_str_buf_cat(buf, p1, p - p1);
}
if (e - p < clen) clen = e - p;
if (clen <= 2) {
clen = 1;
}
else {
const char *q = p;
clen--;
for (; clen > 1; clen--) {
ret = rb_enc_precise_mbclen(q, q + clen, enc);
if (MBCLEN_NEEDMORE_P(ret)) break;
else if (MBCLEN_INVALID_P(ret)) continue;
else UNREACHABLE;
}
}
if (rep) {
rb_str_buf_cat(buf, rep, replen);
if (!rep7bit_p) cr = ENC_CODERANGE_VALID;
}
else {
repl = rb_yield(rb_enc_str_new(p1, clen, enc));
repl = str_compat_and_valid(repl, enc);
rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID)
cr = ENC_CODERANGE_VALID;
}
p += clen;
p1 = p;
p = search_nonascii(p, e);
if (!p) {
p = e;
break;
}
}
else {
UNREACHABLE;
}
}
if (p1 < p) {
rb_str_buf_cat(buf, p1, p - p1);
}
if (p < e) {
if (rep) {
rb_str_buf_cat(buf, rep, replen);
if (!rep7bit_p) cr = ENC_CODERANGE_VALID;
}
else {
repl = rb_yield(rb_enc_str_new(p, e-p, enc));
repl = str_compat_and_valid(repl, enc);
rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID)
cr = ENC_CODERANGE_VALID;
}
}
ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), cr);
return buf;
}
else {
/* ASCII incompatible */
const char *p = RSTRING_PTR(str);
const char *e = RSTRING_END(str);
const char *p1 = p;
VALUE buf = rb_str_buf_new(RSTRING_LEN(str));
const char *rep;
long replen;
long mbminlen = rb_enc_mbminlen(enc);
static rb_encoding *utf16be;
static rb_encoding *utf16le;
static rb_encoding *utf32be;
static rb_encoding *utf32le;
if (!utf16be) {
utf16be = rb_enc_find("UTF-16BE");
utf16le = rb_enc_find("UTF-16LE");
utf32be = rb_enc_find("UTF-32BE");
utf32le = rb_enc_find("UTF-32LE");
}
if (!NIL_P(repl)) {
rep = RSTRING_PTR(repl);
replen = RSTRING_LEN(repl);
}
else if (enc == utf16be) {
rep = "\xFF\xFD";
replen = strlen(rep);
}
else if (enc == utf16le) {
rep = "\xFD\xFF";
replen = strlen(rep);
}
else if (enc == utf32be) {
rep = "\x00\x00\xFF\xFD";
replen = strlen(rep);
}
else if (enc == utf32le) {
rep = "\xFD\xFF\x00\x00";
replen = strlen(rep);
}
else {
rep = "?";
replen = strlen(rep);
}
while (p < e) {
int ret = rb_enc_precise_mbclen(p, e, enc);
if (MBCLEN_NEEDMORE_P(ret)) {
break;
}
else if (MBCLEN_CHARFOUND_P(ret)) {
p += MBCLEN_CHARFOUND_LEN(ret);
}
else if (MBCLEN_INVALID_P(ret)) {
const char *q = p;
long clen = rb_enc_mbmaxlen(enc);
if (p > p1) rb_str_buf_cat(buf, p1, p - p1);
if (e - p < clen) clen = e - p;
if (clen <= mbminlen * 2) {
clen = mbminlen;
}
else {
clen -= mbminlen;
for (; clen > mbminlen; clen-=mbminlen) {
ret = rb_enc_precise_mbclen(q, q + clen, enc);
if (MBCLEN_NEEDMORE_P(ret)) break;
else if (MBCLEN_INVALID_P(ret)) continue;
else UNREACHABLE;
}
}
if (rep) {
rb_str_buf_cat(buf, rep, replen);
}
else {
repl = rb_yield(rb_enc_str_new(p, e-p, enc));
repl = str_compat_and_valid(repl, enc);
rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
}
p += clen;
p1 = p;
}
else {
UNREACHABLE;
}
}
if (p1 < p) {
rb_str_buf_cat(buf, p1, p - p1);
}
if (p < e) {
if (rep) {
rb_str_buf_cat(buf, rep, replen);
}
else {
repl = rb_yield(rb_enc_str_new(p, e-p, enc));
repl = str_compat_and_valid(repl, enc);
rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
}
}
ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), ENC_CODERANGE_VALID);
return buf;
}
}
/**********************************************************************
* Document-class: Symbol
*
@ -8226,6 +8492,7 @@ Init_String(void)
rb_define_method(rb_cString, "getbyte", rb_str_getbyte, 1);
rb_define_method(rb_cString, "setbyte", rb_str_setbyte, 2);
rb_define_method(rb_cString, "byteslice", rb_str_byteslice, -1);
rb_define_method(rb_cString, "scrub", rb_str_scrub, -1);
rb_define_method(rb_cString, "to_i", rb_str_to_i, -1);
rb_define_method(rb_cString, "to_f", rb_str_to_f, 0);

34
test/ruby/test_m17n.rb
View file

@ -1489,4 +1489,38 @@ class TestM17N < Test::Unit::TestCase
s.untrust
assert_equal(true, s.b.untrusted?)
end
def test_scrub
assert_equal("\uFFFD\uFFFD\uFFFD", u("\x80\x80\x80").scrub)
assert_equal("\uFFFDA", u("\xF4\x80\x80A").scrub)
# exapmles in Unicode 6.1.0 D93b
assert_equal("\x41\uFFFD\uFFFD\x41\uFFFD\x41",
u("\x41\xC0\xAF\x41\xF4\x80\x80\x41").scrub)
assert_equal("\x41\uFFFD\uFFFD\uFFFD\x41",
u("\x41\xE0\x9F\x80\x41").scrub)
assert_equal("\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064",
u("\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64").scrub)
assert_equal("abcdefghijklmnopqrstuvwxyz\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064",
u("abcdefghijklmnopqrstuvwxyz\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64").scrub)
assert_equal("\u3042\u3013", u("\xE3\x81\x82\xE3\x81").scrub("\u3013"))
assert_raise(Encoding::CompatibilityError){ u("\xE3\x81\x82\xE3\x81").scrub(e("\xA4\xA2")) }
assert_raise(TypeError){ u("\xE3\x81\x82\xE3\x81").scrub(1) }
assert_raise(ArgumentError){ u("\xE3\x81\x82\xE3\x81\x82\xE3\x81").scrub(u("\x81")) }
assert_equal(e("\xA4\xA2\xA2\xAE"), e("\xA4\xA2\xA4").scrub(e("\xA2\xAE")))
assert_equal("\u3042<e381>", u("\xE3\x81\x82\xE3\x81").scrub{|x|'<'+x.unpack('H*')[0]+'>'})
assert_raise(Encoding::CompatibilityError){ u("\xE3\x81\x82\xE3\x81").scrub{e("\xA4\xA2")} }
assert_raise(TypeError){ u("\xE3\x81\x82\xE3\x81").scrub{1} }
assert_raise(ArgumentError){ u("\xE3\x81\x82\xE3\x81\x82\xE3\x81").scrub{u("\x81")} }
assert_equal(e("\xA4\xA2\xA2\xAE"), e("\xA4\xA2\xA4").scrub{e("\xA2\xAE")})
assert_equal("\uFFFD\u3042".encode("UTF-16BE"),
"\xD8\x00\x30\x42".force_encoding(Encoding::UTF_16BE).
scrub)
assert_equal("\uFFFD\u3042".encode("UTF-16LE"),
"\x00\xD8\x42\x30".force_encoding(Encoding::UTF_16LE).
scrub)
end
end

17
transcode.c
View file

@ -2652,6 +2652,8 @@ str_transcode_enc_args(VALUE str, volatile VALUE *arg1, volatile VALUE *arg2,
return dencidx;
}
VALUE rb_str_scrub(int argc, VALUE *argv, VALUE str);
static int
str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
{
@ -2686,6 +2688,17 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
ECONV_XML_ATTR_CONTENT_DECORATOR|
ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) {
if (senc && senc == denc) {
if (ecflags & ECONV_INVALID_MASK) {
if (!NIL_P(ecopts)) {
VALUE rep = rb_hash_aref(ecopts, sym_replace);
dest = rb_str_scrub(1, &rep, str);
}
else {
dest = rb_str_scrub(0, NULL, str);
}
*self = dest;
return dencidx;
}
return NIL_P(arg2) ? -1 : dencidx;
}
if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
@ -2815,10 +2828,6 @@ static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx);
* in the source encoding. The last form by default does not raise
* exceptions but uses replacement strings.
*
* Please note that conversion from an encoding +enc+ to the
* same encoding +enc+ is a no-op, i.e. the receiver is returned without
* any changes, and no exceptions are raised, even if there are invalid bytes.
*
* The +options+ Hash gives details for conversion and can have the following
* keys:
*