kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

fix each_grapheme_cluster's size [Bug #14363]

Browse source 
From: Hugo Peixoto <hugo.peixoto@gmail.com>

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@62892 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
naruse 2018-03-22 07:58:38 +00:00
parent 6e0f5b8407
commit 41b2ef4685
2 changed files with 61 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
string.c
View file

@ -8355,6 +8355,56 @@ rb_str_codepoints(VALUE str)
return rb_str_enumerate_codepoints(str, ary); return rb_str_enumerate_codepoints(str, ary);
} }
static VALUE
rb_str_each_grapheme_cluster_size(VALUE str, VALUE args, VALUE eobj)
{
long grapheme_cluster_count = 0;
regex_t *reg_grapheme_cluster = NULL;
static regex_t *reg_grapheme_cluster_utf8 = NULL;
int encidx = ENCODING_GET(str);
rb_encoding *enc = rb_enc_from_index(encidx);
int unicode_p = rb_enc_unicode_p(enc);
const char *ptr, *end;
if (!unicode_p || single_byte_optimizable(str)) {
return rb_str_length(str);
}
/* synchronize */
if (encidx == rb_utf8_encindex() && reg_grapheme_cluster_utf8) {
reg_grapheme_cluster = reg_grapheme_cluster_utf8;
}
if (!reg_grapheme_cluster) {
const OnigUChar source[] = "\\X";
int r = onig_new(&reg_grapheme_cluster, source, source + sizeof(source) - 1,
ONIG_OPTION_DEFAULT, enc, OnigDefaultSyntax, NULL);
if (r) {
rb_bug("cannot compile grapheme cluster regexp");
}
if (encidx == rb_utf8_encindex()) {
reg_grapheme_cluster_utf8 = reg_grapheme_cluster;
}
}
ptr = RSTRING_PTR(str);
end = RSTRING_END(str);
while (ptr < end) {
OnigPosition len = onig_match(reg_grapheme_cluster,
(const OnigUChar *)ptr, (const OnigUChar *)end,
(const OnigUChar *)ptr, NULL, 0);
if (len == 0) break;
if (len < 0) {
break;
}
grapheme_cluster_count++;
ptr += len;
}
RB_GC_GUARD(str);
return LONG2NUM(grapheme_cluster_count);
}
static VALUE static VALUE
rb_str_enumerate_grapheme_clusters(VALUE str, VALUE ary) rb_str_enumerate_grapheme_clusters(VALUE str, VALUE ary)
{ {
@ -8426,7 +8476,7 @@ rb_str_enumerate_grapheme_clusters(VALUE str, VALUE ary)
static VALUE static VALUE
rb_str_each_grapheme_cluster(VALUE str) rb_str_each_grapheme_cluster(VALUE str)
{ {
RETURN_SIZED_ENUMERATOR(str, 0, 0, rb_str_each_char_size); RETURN_SIZED_ENUMERATOR(str, 0, 0, rb_str_each_grapheme_cluster_size);
return rb_str_enumerate_grapheme_clusters(str, 0); return rb_str_enumerate_grapheme_clusters(str, 0);
} }

13
test/ruby/test_string.rb
View file

@ -982,11 +982,18 @@ CODE
"\u{1f469 200d 2764 fe0f 200d 1f469}", "\u{1f469 200d 2764 fe0f 200d 1f469}",
].each do |g| ].each do |g|
assert_equal [g], g.each_grapheme_cluster.to_a assert_equal [g], g.each_grapheme_cluster.to_a
assert_equal 1, g.each_grapheme_cluster.size
end
[
["\u{a 308}", ["\u000A", "\u0308"]],
["\u{d 308}", ["\u000D", "\u0308"]],
["abc", ["a", "b", "c"]],
].each do |str, grapheme_clusters|
assert_equal grapheme_clusters, str.each_grapheme_cluster.to_a
assert_equal grapheme_clusters.size, str.each_grapheme_cluster.size
end end
assert_equal ["\u000A", "\u0308"], "\u{a 308}".each_grapheme_cluster.to_a
assert_equal ["\u000D", "\u0308"], "\u{d 308}".each_grapheme_cluster.to_a
assert_equal ["a", "b", "c"], "abc".b.each_grapheme_cluster.to_a
s = ("x"+"\u{10ABCD}"*250000) s = ("x"+"\u{10ABCD}"*250000)
assert_empty(s.each_grapheme_cluster {s.clear}) assert_empty(s.each_grapheme_cluster {s.clear})
end end