kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

* io.c (appendline): should do multibyte aware RS search.

Browse source 
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@14554 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
matz 2007-12-23 19:55:18 +00:00
parent b4f2e2ba07
commit c749064f9f
3 changed files with 23 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ChangeLog
View file

@ -25,6 +25,8 @@ Mon Dec 24 02:59:32 2007 Yukihiro Matsumoto <matz@ruby-lang.org>
* io.c (io_enc_str): should preserve default_external encoding. * io.c (io_enc_str): should preserve default_external encoding.
* io.c (appendline): should do multibyte aware RS search.
Mon Dec 24 02:06:35 2007 Yukihiro Matsumoto <matz@ruby-lang.org> Mon Dec 24 02:06:35 2007 Yukihiro Matsumoto <matz@ruby-lang.org>
* io.c (rb_f_open): documentation update. * io.c (rb_f_open): documentation update.

22
io.c
View file

@ -1647,22 +1647,32 @@ io_read(int argc, VALUE *argv, VALUE io)
} }
static int static int
appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp) appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp, int mb)
{ {
VALUE str = *strp; VALUE str = *strp;
int c = EOF; int c = EOF;
long limit = *lp; long limit = *lp;
rb_encoding *enc = io_read_encoding(fptr);
do { do {
long pending = READ_DATA_PENDING_COUNT(fptr); long pending = READ_DATA_PENDING_COUNT(fptr);
if (pending > 0) { if (pending > 0) {
const char *p = READ_DATA_PENDING_PTR(fptr); const char *s = READ_DATA_PENDING_PTR(fptr);
const char *e; const char *p, *e;
long last = 0, len = (c != EOF); long last = 0, len = (c != EOF);
if (limit > 0 && pending > limit) pending = limit; if (limit > 0 && pending > limit) pending = limit;
p = s;
again:
e = memchr(p, delim, pending); e = memchr(p, delim, pending);
if (e) pending = e - p + 1; if (e) {
if (mb &&
ONIGENC_LEFT_ADJUST_CHAR_HEAD(enc,(UChar*)s,(UChar*)e) != (UChar*)e) {
p = e + 1;
goto again;
}
pending = e - s + 1;
}
len += pending; len += pending;
if (!NIL_P(str)) { if (!NIL_P(str)) {
last = RSTRING_LEN(str); last = RSTRING_LEN(str);
@ -1742,7 +1752,7 @@ rb_io_getline_fast(rb_io_t *fptr, unsigned char delim, long limit)
int c, nolimit = 0; int c, nolimit = 0;
for (;;) { for (;;) {
c = appendline(fptr, delim, &str, &limit); c = appendline(fptr, delim, &str, &limit, 0);
if (c == EOF || c == delim) break; if (c == EOF || c == delim) break;
if (limit == 0) { if (limit == 0) {
nolimit = 1; nolimit = 1;
@ -1842,7 +1852,7 @@ rb_io_getline_1(VALUE rs, long limit, VALUE io)
} }
newline = rsptr[rslen - 1]; newline = rsptr[rslen - 1];
while ((c = appendline(fptr, newline, &str, &limit)) != EOF) { while ((c = appendline(fptr, newline, &str, &limit, 1)) != EOF) {
if (c == newline) { if (c == newline) {
if (RSTRING_LEN(str) < rslen) continue; if (RSTRING_LEN(str) < rslen) continue;
if (!rspara) rscheck(rsptr, rslen, rs); if (!rspara) rscheck(rsptr, rslen, rs);

10
test/ruby/test_io_m17n.rb
View file

@ -60,8 +60,8 @@ EOT
s = open("tmp", "r:euc-jp:utf-8") {|f| s = open("tmp", "r:euc-jp:utf-8") {|f|
f.gets("\xA2\xA2".force_encoding("euc-jp").encode("utf-8")) f.gets("\xA2\xA2".force_encoding("euc-jp").encode("utf-8"))
} }
assert_equal(Encoding.find("euc-jp"), s.encoding) assert_equal(Encoding.find("utf-8"), s.encoding)
assert_str_equal("before \xA1\xA2\xA2\xA3 after".force_encoding("iso-8859-1"), s, '[ruby-core:14319]') assert_str_equal("before \xA1\xA2\xA2\xA3 after".force_encoding("euc-jp").encode("utf-8"), s, '[ruby-core:14319]')
} }
end end
@ -196,8 +196,6 @@ EOT
w.close w.close
s = r.read s = r.read
assert_equal(Encoding.default_external, s.encoding) assert_equal(Encoding.default_external, s.encoding)
puts encdump(s)
puts encdump(utf8)
assert_str_equal(utf8, s) assert_str_equal(utf8, s)
} }
@ -226,7 +224,9 @@ EOT
} }
} }
ENCS.reject {|e| e == Encoding::ASCII_8BIT }.each {|enc| ENCS.each {|enc|
next if enc == Encoding::ASCII_8BIT
next if enc == Encoding::UTF_8
with_pipe("#{enc}:UTF-8") {|r, w| with_pipe("#{enc}:UTF-8") {|r, w|
w << "\xc2\xa1" w << "\xc2\xa1"
w.close w.close