* encoding.c: provide basic features for M17N.

* parse.y: encoding aware parsing.

* parse.y (pragma_encoding): encoding specification pragma.

* parse.y (rb_intern3): encoding specified symbols.

* string.c (rb_str_length): length based on characters.  
  for older behavior, bytesize method added.

* string.c (rb_str_index_m): index based on characters.  rindex as
  well.

* string.c (succ_char): encoding aware succeeding string.

* string.c (rb_str_reverse): reverse based on characters.

* string.c (rb_str_inspect): encoding aware string description.

* string.c (rb_str_upcase_bang): encoding aware case conversion.
  downcase, capitalize, swapcase as well.

* string.c (rb_str_tr_bang): tr based on characters.  delete,
  squeeze, tr_s, count as well.

* string.c (rb_str_split_m): split based on characters.

* string.c (rb_str_each_line): encoding aware each_line.

* string.c (rb_str_each_char): added.  iteration based on
  characters.

* string.c (rb_str_strip_bang): encoding aware whitespace
  stripping.  lstrip, rstrip as well.

* string.c (rb_str_justify): encoding aware justifying (ljust,
  rjust, center).

* string.c (str_encoding): get encoding attribute from a string. 

* re.c (rb_reg_initialize): encoding aware regular expression

* sprintf.c (rb_str_format): formatting (i.e. length count) based
  on characters.

* io.c (rb_io_getc): getc to return one-character string.
  for older behavior, getbyte method added.

* ext/stringio/stringio.c (strio_getc): ditto.

* io.c (rb_io_ungetc): allow pushing arbitrary string at the
  current reading point.

* ext/stringio/stringio.c (strio_ungetc): ditto.

* ext/strscan/strscan.c: encoding support.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@13261 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,63 @@
+Sat Aug 25 11:45:37 2007  Yukihiro Matsumoto  <matz@ruby-lang.org>
+
+	* encoding.c: provide basic features for M17N.
+
+	* parse.y: encoding aware parsing.
+
+	* parse.y (pragma_encoding): encoding specification pragma.
+
+	* parse.y (rb_intern3): encoding specified symbols.
+
+	* string.c (rb_str_length): length based on characters.  
+	  for older behavior, bytesize method added.
+
+	* string.c (rb_str_index_m): index based on characters.  rindex as
+	  well.
+
+	* string.c (succ_char): encoding aware succeeding string.
+
+	* string.c (rb_str_reverse): reverse based on characters.
+
+	* string.c (rb_str_inspect): encoding aware string description.
+
+	* string.c (rb_str_upcase_bang): encoding aware case conversion.
+	  downcase, capitalize, swapcase as well.
+
+	* string.c (rb_str_tr_bang): tr based on characters.  delete,
+	  squeeze, tr_s, count as well.
+
+	* string.c (rb_str_split_m): split based on characters.
+
+	* string.c (rb_str_each_line): encoding aware each_line.
+
+	* string.c (rb_str_each_char): added.  iteration based on
+	  characters.
+
+	* string.c (rb_str_strip_bang): encoding aware whitespace
+	  stripping.  lstrip, rstrip as well.
+
+	* string.c (rb_str_justify): encoding aware justifying (ljust,
+	  rjust, center).
+
+	* string.c (str_encoding): get encoding attribute from a string. 
+
+	* re.c (rb_reg_initialize): encoding aware regular expression
+
+	* sprintf.c (rb_str_format): formatting (i.e. length count) based
+	  on characters.
+
+	* io.c (rb_io_getc): getc to return one-character string.
+	  for older behavior, getbyte method added.
+
+	* ext/stringio/stringio.c (strio_getc): ditto.
+
+	* io.c (rb_io_ungetc): allow pushing arbitrary string at the
+	  current reading point.
+
+	* ext/stringio/stringio.c (strio_ungetc): ditto.
+
+	* ext/strscan/strscan.c: encoding support.
+
 Sat Aug 25 10:59:19 2007  Koichi Sasada  <ko1@atdot.net>
 
 	* cont.c: separate Continuation and Fiber from core.

array.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Aug  6 09:46:12 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

bignum.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Jun 10 00:48:55 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

class.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Aug 10 15:05:44 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

common.mk

@@ -25,6 +25,7 @@ OBJS	      = array.$(OBJEXT) \
 		compar.$(OBJEXT) \
 		dir.$(OBJEXT) \
 		dln.$(OBJEXT) \
+		encoding.$(OBJEXT) \
 		enum.$(OBJEXT) \
 		enumerator.$(OBJEXT) \
 		error.$(OBJEXT) \
@@ -401,6 +402,7 @@ dmydln.$(OBJEXT): {$(VPATH)}dmydln.c {$(VPATH)}dln.c {$(VPATH)}ruby.h \
   {$(VPATH)}config.h {$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h \
   {$(VPATH)}dln.h
 dmyext.$(OBJEXT): {$(VPATH)}dmyext.c
+encoding.$(OBJEXT): {$(VPATH)}encoding.c {$(VPATH)}encoding.h 
 enum.$(OBJEXT): {$(VPATH)}enum.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
   {$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h \
   {$(VPATH)}node.h {$(VPATH)}util.h
@@ -523,7 +525,7 @@ sprintf.$(OBJEXT): {$(VPATH)}sprintf.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
 st.$(OBJEXT): {$(VPATH)}st.c {$(VPATH)}config.h {$(VPATH)}st.h {$(VPATH)}defines.h
 string.$(OBJEXT): {$(VPATH)}string.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
   {$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h \
-  {$(VPATH)}re.h {$(VPATH)}regex.h
+  {$(VPATH)}re.h {$(VPATH)}regex.h {$(VPATH)}encoding.h 
 struct.$(OBJEXT): {$(VPATH)}struct.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
   {$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h
 thread.$(OBJEXT): {$(VPATH)}thread.c {$(VPATH)}eval_intern.h \

compar.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Aug 26 14:39:48 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

dir.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Wed Jan  5 09:51:01 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

dln.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Jan 18 17:05:06 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

dln.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Wed Jan 19 16:53:09 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

enum.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Oct  1 15:15:19 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

error.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Mon Aug  9 16:11:34 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

euc_jp.c

@@ -78,8 +78,9 @@ static int
 code_to_mbclen(OnigCodePoint code)
 {
   if (ONIGENC_IS_CODE_ASCII(code)) return 1;
-  else if ((code & 0xff0000) != 0) return 3;
+  else if (code > 0xffffff) return 0;
-  else if ((code &   0xff00) != 0) return 2;
+  else if ((code & 0xff0000) >= 0x800000) return 3;
+  else if ((code &   0xff00) >= 0x8000) return 2;
   else return 0;
 }
 

eval.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Jun 10 14:22:17 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

ext/socket/socket.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Mar 31 12:21:29 JST 1994
 
-  Copyright (C) 1993-2001 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 ************************************************/
 

ext/stringio/stringio.c

@@ -13,7 +13,7 @@
 **********************************************************************/
 
 #include "ruby.h"
-#include "rubyio.h"
+#include "ruby/io.h"
 #if defined(HAVE_FCNTL_H) || defined(_WIN32)
 #include <fcntl.h>
 #elif defined(HAVE_SYS_FCNTL_H)
@@ -84,6 +84,18 @@ get_strio(VALUE self)
     return ptr;
 }
 
+static VALUE
+strio_substr(struct StringIO *ptr, int pos, int len)
+{
+    VALUE str = ptr->string;
+    rb_encoding *enc = rb_enc_get(str);
+    int rlen = RSTRING_LEN(str) - pos;
+
+    if (len > rlen) len = rlen;
+    if (len < 0) len = 0;
+    return rb_enc_str_new(RSTRING_PTR(str)+pos, len, enc);
+}
+
 #define StringIO(obj) get_strio(obj)
 
 #define CLOSED(ptr) (!((ptr)->flags & FMODE_READWRITE))
@@ -603,7 +615,7 @@ strio_each_byte(VALUE self)
 
 /*
  * call-seq:
- *   strio.getc   -> fixnum or nil
+ *   strio.getc   -> string or nil
  *
  * See IO#getc.
  */
@@ -611,15 +623,17 @@ static VALUE
 strio_getc(VALUE self)
 {
     struct StringIO *ptr = readable(StringIO(self));
-    int c;
+    rb_encoding *enc = rb_enc_get(ptr->string);
-    char ch;
+    int len;
+    char *p;
 
     if (ptr->pos >= RSTRING_LEN(ptr->string)) {
 	return Qnil;
     }
-    c = RSTRING_PTR(ptr->string)[ptr->pos++];
+    p = RSTRING_PTR(ptr->string)+ptr->pos;
-    ch = c & 0xff;
+    len = rb_enc_mbclen(p, enc);
-    return rb_str_new(&ch, 1);
+    ptr->pos += len;
+    return rb_enc_str_new(p, len, rb_enc_get(ptr->string));
 }
 
 /*
@@ -671,30 +685,34 @@ static VALUE
 strio_ungetc(VALUE self, VALUE c)
 {
     struct StringIO *ptr = readable(StringIO(self));
-    int cc;
+    long lpos, clen;
-    long len, pos = ptr->pos;
+    char *p, *pend;
+    rb_encoding *enc;
 
     if (NIL_P(c)) return Qnil;
     if (FIXNUM_P(c)) {
-	cc = FIX2INT(c);
+	int cc = FIX2INT(c);
+	char buf[16];
+
+	enc = rb_enc_get(ptr->string);
+	rb_enc_mbcput(cc, buf, enc);
+	c = rb_enc_str_new(buf, rb_enc_codelen(cc, enc), enc);
     }
     else {
 	SafeStringValue(c);
-	if (RSTRING_LEN(c) > 1) {
+	enc = rb_enc_check(ptr->string, c);
-	    rb_warn("IO#ungetc pushes back only one byte");
-	}
-	cc = (unsigned char)RSTRING_PTR(c)[0];
     }
-    if (cc != EOF && pos > 0) {
+    /* get logical position */
-	if ((len = RSTRING_LEN(ptr->string)) < pos-- ||
+    lpos = 0; p = RSTRING_PTR(ptr->string); pend = p + ptr->pos - 1;
-	    (unsigned char)RSTRING_PTR(ptr->string)[pos] !=
+    for (;;) {
-	    (unsigned char)cc) {
+	clen = rb_enc_mbclen(p, enc);
-	    strio_extend(ptr, pos, 1);
+	if (p+clen >= pend) break;
-	    RSTRING_PTR(ptr->string)[pos] = cc;
+	p += clen;
-	    OBJ_INFECT(ptr->string, self);
+	lpos++;
-	}
-	--ptr->pos;
     }
+    rb_str_update(ptr->string, lpos, ptr->pos ? 1 : 0, c);
+    ptr->pos = p - RSTRING_PTR(ptr->string);
+
     return Qnil;
 }
 
@@ -800,7 +818,7 @@ strio_getline(int argc, VALUE *argv, struct StringIO *ptr)
 	e = s + limit;
     }
     if (NIL_P(str)) {
-	str = rb_str_substr(ptr->string, ptr->pos, e - s);
+	str = strio_substr(ptr, ptr->pos, e - s);
     }
     else if ((n = RSTRING_LEN(str)) == 0) {
 	p = s;
@@ -816,13 +834,13 @@ strio_getline(int argc, VALUE *argv, struct StringIO *ptr)
 		break;
 	    }
 	}
-	str = rb_str_substr(ptr->string, s - RSTRING_PTR(ptr->string), e - s); 
+	str = strio_substr(ptr, s - RSTRING_PTR(ptr->string), e - s); 
     }
     else if (n == 1) {
 	if ((p = memchr(s, RSTRING_PTR(str)[0], e - s)) != 0) {
 	    e = p + 1;
 	}
-	str = rb_str_substr(ptr->string, ptr->pos, e - s);
+	str = strio_substr(ptr, ptr->pos, e - s);
     }
     else {
 	if (n < e - s) {
@@ -843,7 +861,7 @@ strio_getline(int argc, VALUE *argv, struct StringIO *ptr)
 		}
 	    }
 	}
-	str = rb_str_substr(ptr->string, ptr->pos, e - s);
+	str = strio_substr(ptr, ptr->pos, e - s);
     }
     ptr->pos = e - RSTRING_PTR(ptr->string);
     ptr->lineno++;
@@ -944,7 +962,7 @@ strio_write(VALUE self, VALUE str)
     if (TYPE(str) != T_STRING)
 	str = rb_obj_as_string(str);
     len = RSTRING_LEN(str);
-    if (!len) return INT2FIX(0);
+    if (len == 0) return INT2FIX(0);
     check_modifiable(ptr);
     olen = RSTRING_LEN(ptr->string);
     if (ptr->flags & FMODE_APPEND) {
@@ -955,7 +973,8 @@ strio_write(VALUE self, VALUE str)
     }
     else {
 	strio_extend(ptr, ptr->pos, len);
-	rb_str_update(ptr->string, ptr->pos, len, str);
+	memmove(RSTRING_PTR(ptr->string)+ptr->pos, RSTRING_PTR(str), len);
+	OBJ_INFECT(ptr->string, str);
     }
     OBJ_INFECT(ptr->string, self);
     ptr->pos += len;
@@ -1070,7 +1089,7 @@ strio_read(int argc, VALUE *argv, VALUE self)
 	rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc);
     }
     if (NIL_P(str)) {
-	str = rb_str_substr(ptr->string, ptr->pos, len);
+	str = strio_substr(ptr, ptr->pos, len);
     }
     else {
 	long rest = RSTRING_LEN(ptr->string) - ptr->pos;

ext/strscan/strscan.c

@@ -10,6 +10,7 @@
 
 #include "ruby/ruby.h"
 #include "ruby/re.h"
+#include "ruby/encoding.h"
 
 #define STRSCAN_VERSION "0.7.0"
 
@@ -189,6 +190,7 @@ strscan_initialize(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &str, &need_dup);
     StringValue(str);
     p->str = str;
+    rb_enc_associate(self, rb_enc_get(str));
 
     return self;
 }
@@ -652,13 +654,14 @@ strscan_getch(VALUE self)
 {
     struct strscanner *p;
     long len;
+    rb_encoding *enc = rb_enc_get(self);
 
     GET_SCANNER(self, p);
     CLEAR_MATCH_STATUS(p);
     if (EOS_P(p))
         return Qnil;
 
-    len = mbclen(*CURPTR(p));
+    len = rb_enc_mbclen(CURPTR(p), enc);
     if (p->curr + len > S_LEN(p)) {
         len = S_LEN(p) - p->curr;
     }

ext/syck/emitter.c

@@ -7,7 +7,7 @@
  * Copyright (C) 2003 why the lucky stiff
  * 
  * All Base64 code from Ruby's pack.c.
- * Ruby is Copyright (C) 1993-2003 Yukihiro Matsumoto 
+ * Ruby is Copyright (C) 1993-2007 Yukihiro Matsumoto 
  */
 #include "ruby/ruby.h"
 

ext/tk/sample/demos-jp/anilabel.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # animated label widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/aniwave.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # animated wave demo (called by 'widget')
 #

ext/tk/sample/demos-jp/arrow.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # arrowhead widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/bind.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # text (tag bindings) widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/bitmap.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # bitmap widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/button.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # button widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/check.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # checkbutton widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/check2.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # checkbutton widget demo2 (called by 'widget')
 #

ext/tk/sample/demos-jp/clrpick.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # widget demo prompts the user to select a color (called by 'widget')
 #

ext/tk/sample/demos-jp/colors.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # listbox widget demo 'colors' (called by 'widget')
 #

ext/tk/sample/demos-jp/cscroll.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # simple scrollable canvas widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/ctext.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # Canvas Text widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/dialog1.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # a dialog box with a local grab (called by 'widget')
 #

ext/tk/sample/demos-jp/dialog2.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # a dialog box with a global grab (called by 'widget')
 #

ext/tk/sample/demos-jp/entry1.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # entry (no scrollbars) widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/entry2.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # entry (with scrollbars) widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/entry3.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 # entry3.rb --
 #
 # This demonstration script creates several entry widgets whose

ext/tk/sample/demos-jp/filebox.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # widget demo prompts the user to select a file (called by 'widget')
 #

ext/tk/sample/demos-jp/floor.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # floorDisplay widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/floor2.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # floorDisplay widget demo 2 (called by 'widget')
 #

ext/tk/sample/demos-jp/form.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # form widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/goldberg.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # Ruby/Tk Goldverg demo (called by 'widget')
 #

ext/tk/sample/demos-jp/hello

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# -*- coding: euc-jp -*-
 require 'tk'
 
 TkButton.new(nil,

ext/tk/sample/demos-jp/hscale.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 require "tkcanvas"
 
 if defined?($hscale_demo) && $hscale_deom

ext/tk/sample/demos-jp/icon.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # iconic button widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/image1.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # two image widgets demo (called by 'widget')
 #

ext/tk/sample/demos-jp/image2.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # widget demo 'load image' (called by 'widget')
 #

ext/tk/sample/demos-jp/image3.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 # image3.rb
 #
 # This demonstration script creates a simple collection of widgets

ext/tk/sample/demos-jp/items.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # canvas item types widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/ixset2

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# -*- coding: euc-jp -*-
 #
 # ixset --
 # A nice interface to "xset" to change X server settings

ext/tk/sample/demos-jp/label.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # label widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/labelframe.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # labelframe.rb
 #
 # This demonstration script creates a toplevel window containing

ext/tk/sample/demos-jp/menu.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # menus widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/menu84.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # menus widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/menu8x.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # menus widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/menubu.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 require "tkcanvas"
 
 def optionMenu(menubutton, varName, firstValue, *rest)

ext/tk/sample/demos-jp/msgbox.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # message boxes widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/paned1.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # paned1.rb
 #
 # This demonstration script creates a toplevel window containing

ext/tk/sample/demos-jp/paned2.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # paned2.rb --
 #
 # This demonstration script creates a toplevel window containing

ext/tk/sample/demos-jp/pendulum.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # This demonstration illustrates how Tcl/Tk can be used to construct
 # simulations of physical systems.

ext/tk/sample/demos-jp/plot.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # 2-D plot widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/puzzle.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # widet demo 'puzzle' (called by 'widget')
 #

ext/tk/sample/demos-jp/radio.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # radiobutton widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/radio2.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # radio2.rb
 #
 # This demonstration script creates a toplevel window containing

ext/tk/sample/demos-jp/radio3.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # radio3.rb
 #
 # This demonstration script creates a toplevel window containing

ext/tk/sample/demos-jp/rolodex-j

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# -*- coding: euc-jp -*-
 #
 # rolodex --
 # このスクリプトは Tom LaStrange の rolodex の一部です。

ext/tk/sample/demos-jp/ruler.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # ruler widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/sayings.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # listbox widget demo 'sayings' (called by 'widget')
 #

ext/tk/sample/demos-jp/search.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # Text Search widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/spin.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # spin.rb --
 #
 # This demonstration script creates several spinbox widgets.

ext/tk/sample/demos-jp/states.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # listbox widget demo 'states' (called by 'widget')
 #

ext/tk/sample/demos-jp/style.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # text (display styles) widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/tcolor

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# -*- coding: euc-jp -*-
 #
 # tcolor --
 # このスクリプトはRGB,HSB,CYM形式をサポートする

ext/tk/sample/demos-jp/text.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # text (basic facilities) widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/twind.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # text (embedded windows) widget demo (called by 'widget')
 #

ext/tk/sample/demos-jp/twind2.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 #
 # text (embedded windows) widget demo 2 (called by 'widget')
 #

ext/tk/sample/demos-jp/unicodeout.rb

@@ -1,3 +1,5 @@
+# -*- coding: euc-jp -*-
+#
 # unicodeout.rb --
 #
 # This demonstration script shows how you can produce output (in label

ext/tk/sample/demos-jp/vscale.rb

@@ -1,3 +1,4 @@
+# -*- coding: euc-jp -*-
 require "tkcanvas"
 
 if defined?($vscale_demo) && $vscale_demo

ext/tk/sample/demos-jp/widget

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# -*- coding: euc-jp -*-
 
 # 漢字コード設定 ( tk.rb のロード時の encoding 推定/設定に使われる )
 $KCODE = 'euc'

ext/tk/sample/tkextlib/vu/canvSticker2.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
 
 require 'tk'
 require 'tkextlib/vu/charts'

file.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Mon Nov 15 12:24:34 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

gc.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Oct  5 09:44:46 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

hash.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Mon Nov 22 18:51:18 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

include/ruby/intern.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Jun 10 14:22:17 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 
@@ -337,7 +337,7 @@ RUBY_EXTERN VALUE rb_default_rs;
 RUBY_EXTERN VALUE rb_output_rs;
 VALUE rb_io_write(VALUE, VALUE);
 VALUE rb_io_gets(VALUE);
-VALUE rb_io_getc(VALUE);
+VALUE rb_io_getbyte(VALUE);
 VALUE rb_io_ungetc(VALUE, VALUE);
 VALUE rb_io_close(VALUE);
 VALUE rb_io_flush(VALUE);
@@ -444,7 +444,7 @@ VALUE rb_reg_last_match(VALUE);
 VALUE rb_reg_match_pre(VALUE);
 VALUE rb_reg_match_post(VALUE);
 VALUE rb_reg_match_last(VALUE);
-VALUE rb_reg_new(const char*, long, int);
+VALUE rb_reg_new(VALUE, int);
 VALUE rb_reg_match(VALUE, VALUE);
 VALUE rb_reg_match2(VALUE);
 int rb_reg_options(VALUE);
@@ -498,6 +498,7 @@ VALUE rb_str_unlocktmp(VALUE);
 VALUE rb_str_dup_frozen(VALUE);
 VALUE rb_str_plus(VALUE, VALUE);
 VALUE rb_str_times(VALUE, VALUE);
+int rb_str_sublen(VALUE, int);
 VALUE rb_str_substr(VALUE, long, long);
 void rb_str_modify(VALUE);
 VALUE rb_str_freeze(VALUE);

include/ruby/io.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Nov 12 16:47:09 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
@@ -22,6 +22,7 @@ extern "C" {
 
 #include <stdio.h>
 #include <errno.h>
+#include "ruby/encoding.h"
 
 #if defined(HAVE_STDIO_EXT_H)
 #include <stdio_ext.h>
@@ -44,6 +45,7 @@ typedef struct rb_io_t {
     int rbuf_off;
     int rbuf_len;
     int rbuf_capa;
+    rb_encoding *enc;
 } rb_io_t;
 
 #define HAVE_RB_IO_T 1

include/ruby/node.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri May 28 15:14:02 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

include/ruby/re.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Sep 30 14:18:32 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

include/ruby/regex.h

@@ -5,7 +5,7 @@
   $Author$
   $Date$
 
-  Copyright (C) 1993-2005 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
@@ -29,10 +29,8 @@ extern "C" {
 
 ONIG_EXTERN OnigEncoding    OnigEncDefaultCharEncoding;
 
-#undef ismbchar
+#define ismbchar(p, enc) (mbclen((p),(enc)) != 1)
-#define ismbchar(c) (mbclen((c)) != 1)
+#define mbclen(p,enc)  rb_enc_mbclen((p), (enc))
-#define mbclen(c)  \
-  ONIGENC_MBC_ENC_LEN(OnigEncDefaultCharEncoding, (UChar* )(&c))
 
 #endif /* ifndef ONIG_RUBY_M17N */
 

include/ruby/ruby.h

@@ -5,7 +5,7 @@
   $Author$
   created at: Thu Jun 10 14:26:32 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 
@@ -455,6 +455,7 @@ struct RString {
     (!(RBASIC(str)->flags & RSTRING_NOEMBED) ? \
      RSTRING(str)->as.ary : \
      RSTRING(str)->as.heap.ptr)
+#define RSTRING_END(str) (RSTRING_PTR(str)+RSTRING_LEN(str))
 
 struct RArray {
     struct RBasic basic;
@@ -598,6 +599,32 @@ enum ruby_value_flags {
 #define FL_USER6     RUBY_FL_USER6
     RUBY_FL_USER7     = (1<<(FL_USHIFT+7)),
 #define FL_USER7     RUBY_FL_USER7
+    RUBY_FL_USER8     = (1<<(FL_USHIFT+8)),
+#define FL_USER8     RUBY_FL_USER8
+    RUBY_FL_USER9     = (1<<(FL_USHIFT+9)),
+#define FL_USER9     RUBY_FL_USER9
+    RUBY_FL_USER10     = (1<<(FL_USHIFT+10)),
+#define FL_USER10     RUBY_FL_USER10
+    RUBY_FL_USER11     = (1<<(FL_USHIFT+11)),
+#define FL_USER11     RUBY_FL_USER11
+    RUBY_FL_USER12     = (1<<(FL_USHIFT+12)),
+#define FL_USER12     RUBY_FL_USER12
+    RUBY_FL_USER13     = (1<<(FL_USHIFT+13)),
+#define FL_USER13     RUBY_FL_USER13
+    RUBY_FL_USER14     = (1<<(FL_USHIFT+14)),
+#define FL_USER14     RUBY_FL_USER14
+    RUBY_FL_USER15     = (1<<(FL_USHIFT+15)),
+#define FL_USER15     RUBY_FL_USER15
+    RUBY_FL_USER16     = (1<<(FL_USHIFT+16)),
+#define FL_USER16     RUBY_FL_USER16
+    RUBY_FL_USER17     = (1<<(FL_USHIFT+17)),
+#define FL_USER17     RUBY_FL_USER17
+    RUBY_FL_USER18     = (1<<(FL_USHIFT+18)),
+#define FL_USER18     RUBY_FL_USER18
+    RUBY_FL_USER19     = (1<<(FL_USHIFT+19)),
+#define FL_USER19     RUBY_FL_USER19
+    RUBY_FL_USER20     = (1<<(FL_USHIFT+20)),
+#define FL_USER20     RUBY_FL_USER20
 };
 
 #define SPECIAL_CONST_P(x) (IMMEDIATE_P(x) || !RTEST(x))
@@ -667,6 +694,7 @@ void rb_gc_unregister_address(VALUE*);
 
 ID rb_intern(const char*);
 ID rb_intern2(const char*, long);
+ID rb_intern_str(VALUE str);
 const char *rb_id2name(ID);
 ID rb_to_id(VALUE);
 VALUE rb_id2str(ID);

include/ruby/signal.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Wed Aug 16 01:15:38 JST 1995
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

include/ruby/util.h

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Mar  9 11:55:53 JST 1995
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

inits.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Dec 28 16:01:58 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

insns.def

@@ -406,7 +406,7 @@ toregexp
 (VALUE val)
 {
     volatile VALUE tmp = str; /* for GC */
-    val = rb_reg_new(RSTRING_PTR(str), RSTRING_LEN(str), flag);
+    val = rb_reg_new(str, flag);
 }
 
 /**

io.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Oct 15 18:08:59 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 
@@ -278,27 +278,38 @@ io_unread(rb_io_t *fptr)
     return;
 }
 
-static int
+static void
-io_ungetc(int c, rb_io_t *fptr)
+io_ungetc(VALUE str, rb_io_t *fptr)
 {
+    int len = RSTRING_LEN(str);
+
     if (fptr->rbuf == NULL) {
         fptr->rbuf_off = 0;
         fptr->rbuf_len = 0;
-        fptr->rbuf_capa = 8192;
+	if (len > 8192)
+	    fptr->rbuf_capa = len;
+	else
+	    fptr->rbuf_capa = 8192;
         fptr->rbuf = ALLOC_N(char, fptr->rbuf_capa);
     }
-    if (c < 0 || fptr->rbuf_len == fptr->rbuf_capa) {
-        return -1;
-    }
     if (fptr->rbuf_off == 0) {
-        if (fptr->rbuf_len)
+        if (fptr->rbuf_len) {
-            MEMMOVE(fptr->rbuf+1, fptr->rbuf, char, fptr->rbuf_len);
+            MEMMOVE(fptr->rbuf+len, fptr->rbuf, char, fptr->rbuf_len);
-        fptr->rbuf_off = 1;
+	}
+        fptr->rbuf_off = len;
     }
-    fptr->rbuf_off--;
+    else if (fptr->rbuf_off < len) {
-    fptr->rbuf_len++;
+	int capa = fptr->rbuf_len + len;
-    fptr->rbuf[fptr->rbuf_off] = c;
+	char *buf = ALLOC_N(char, capa);
-    return c;
+
+        if (fptr->rbuf_len) {
+            MEMMOVE(buf+len, fptr->rbuf+fptr->rbuf_off, char, fptr->rbuf_len);
+	}
+	fptr->rbuf_off = len;
+    }
+    fptr->rbuf_off-=len;
+    fptr->rbuf_len+=len;
+    MEMMOVE(fptr->rbuf+fptr->rbuf_off, RSTRING_PTR(str), char, len);
 }
 
 static rb_io_t *
@@ -875,16 +886,10 @@ rb_io_rewind(VALUE io)
 }
 
 static int
-io_getc(rb_io_t *fptr)
+io_fillbuf(rb_io_t *fptr)
 {
     int r;
-    if (fptr->fd == 0 && (fptr->mode & FMODE_TTY) && TYPE(rb_stdout) == T_FILE) {
+
-        rb_io_t *ofp;
-        GetOpenFile(rb_stdout, ofp);
-        if (ofp->mode & FMODE_TTY) {
-            rb_io_flush(rb_stdout);
-        }
-    }
     if (fptr->rbuf == NULL) {
         fptr->rbuf_off = 0;
         fptr->rbuf_len = 0;
@@ -906,9 +911,7 @@ io_getc(rb_io_t *fptr)
         if (r == 0)
             return -1; /* EOF */
     }
-    fptr->rbuf_off++;
+    return 0;
-    fptr->rbuf_len--;
-    return (unsigned char)fptr->rbuf[fptr->rbuf_off-1];
 }
 
 /*
@@ -947,20 +950,16 @@ VALUE
 rb_io_eof(VALUE io)
 {
     rb_io_t *fptr;
-    int ch;
 
     GetOpenFile(io, fptr);
     rb_io_check_readable(fptr);
 
     if (READ_DATA_PENDING(fptr)) return Qfalse;
     READ_CHECK(fptr);
-    ch = io_getc(fptr);
+    if (io_fillbuf(fptr) < 0) {
-
+	return Qtrue;
-    if (ch != EOF) {
-	io_ungetc(ch, fptr);
-	return Qfalse;
     }
-    return Qtrue;
+    return Qfalse;
 }
 
 /*
@@ -1167,13 +1166,9 @@ io_fread(VALUE str, long offset, rb_io_t *fptr)
 	}
 	rb_thread_wait_fd(fptr->fd);
 	rb_io_check_closed(fptr);
-	c = io_getc(fptr);
+	if (io_fillbuf(fptr) < 0) {
-	if (c < 0) {
 	    break;
 	}
-	RSTRING_PTR(str)[offset++] = c;
-	if (offset > RSTRING_LEN(str)) break;
-	n--;
     }
     return len - n;
 }
@@ -1599,9 +1594,7 @@ appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp)
 	}
 	rb_thread_wait_fd(fptr->fd);
 	rb_io_check_closed(fptr);
-	c = io_getc(fptr);
+	if (io_fillbuf(fptr) < 0) {
-	limit--;
-	if (c < 0) {
 	    *lp = limit;
 	    return c;
 	}
@@ -1640,10 +1633,8 @@ swallow(rb_io_t *fptr, int term)
 	}
 	rb_thread_wait_fd(fptr->fd);
 	rb_io_check_closed(fptr);
-	c = io_getc(fptr);
+	if (io_fillbuf(fptr) < 0) {
-	if (c != term) {
+	    break;
-	    io_ungetc(c, fptr);
-	    return Qtrue;
 	}
     } while (c != EOF);
     return Qfalse;
@@ -2020,20 +2011,24 @@ static VALUE
 rb_io_each_byte(VALUE io)
 {
     rb_io_t *fptr;
-    int c;
+    char *p, *e;
 
     RETURN_ENUMERATOR(io, 0, 0);
     GetOpenFile(io, fptr);
 
     for (;;) {
+	p = fptr->rbuf+fptr->rbuf_off;
+	e = p + fptr->rbuf_len;
+	while (p < e) {
+	    rb_yield(INT2FIX(*p & 0xff));
+	    p++;
+	}
 	rb_io_check_readable(fptr);
 	READ_CHECK(fptr);
-	c = io_getc(fptr);
+	if (io_fillbuf(fptr) < 0) {
-	if (c < 0) {
 	    break;
 	}
-	rb_yield(INT2FIX(c & 0xff));
+   }
-    }
     return io;
 }
 
@@ -2070,54 +2065,54 @@ rb_io_bytes(VALUE str)
     return rb_enumeratorize(str, ID2SYM(rb_intern("each_byte")), 0, 0);
 }
 
-VALUE
-rb_io_getc(VALUE io)
-{
-    rb_io_t *fptr;
-    int c;
-
-    GetOpenFile(io, fptr);
-    rb_io_check_readable(fptr);
-
-    READ_CHECK(fptr);
-    c = io_getc(fptr);
-
-    if (c < 0) {
-	return Qnil;
-    }
-    return INT2FIX(c & 0xff);
-}
-
 /*
  *  call-seq:
- *     ios.getc   => string or nil
+ *     ios.getc   => fixnum or nil
- *
+ *  
  *  Reads a one-character string from <em>ios</em>. Returns
  *  <code>nil</code> if called at end of file.
- *
+ *     
  *     f = File.new("testfile")
  *     f.getc   #=> "8"
  *     f.getc   #=> "1"
  */
 
-VALUE
+static VALUE
-rb_io_getc_m(VALUE io)
+rb_io_getc(VALUE io)
 {
-    char ch;
+    rb_encoding *enc;
     rb_io_t *fptr;
-    int c;
+    int n, left;
+    VALUE str;
 
     GetOpenFile(io, fptr);
     rb_io_check_readable(fptr);
+    enc = rb_enc_get(io);
 
     READ_CHECK(fptr);
-    c = io_getc(fptr);
+    if (io_fillbuf(fptr) < 0) {
-
+	rb_eof_error();
-    if (c < 0) {
-	return Qnil;
     }
-    ch = c & 0xff; 
+    n = rb_enc_mbclen(fptr->rbuf+fptr->rbuf_off, enc);
-    return rb_str_new(&ch, 1);
+    if (n < fptr->rbuf_len) {
+	str = rb_str_new(fptr->rbuf+fptr->rbuf_off, n);
+	fptr->rbuf_off += n;
+	fptr->rbuf_len -= n;
+    }
+    else {
+	str = rb_str_new(0, n);
+	left = fptr->rbuf_len;
+	MEMCPY(RSTRING_PTR(str), fptr->rbuf+fptr->rbuf_off, char, left);
+	if (io_fillbuf(fptr) < 0) {
+	    rb_eof_error();
+	}
+	MEMCPY(RSTRING_PTR(str)+left, fptr->rbuf, char, n-left);
+	fptr->rbuf_off += left;
+	fptr->rbuf_len -= left;
+    }
+    rb_enc_associate(str, enc);
+
+    return str;
 }
 
 int
@@ -2139,14 +2134,74 @@ rb_getc(FILE *f)
  *  call-seq:
  *     ios.readchar   => string
  *
- *  Reads a character as with <code>IO#getc</code>, but raises an
+ *  Reads a one-character string from <em>ios</em>. Raises an
  *  <code>EOFError</code> on end of file.
+ *
+ *     f = File.new("testfile")
+ *     f.readchar   #=> "8"
+ *     f.readchar   #=> "1"
  */
 
 static VALUE
 rb_io_readchar(VALUE io)
 {
-    VALUE c = rb_io_getc_m(io);
+    VALUE c = rb_io_getc(io);
+
+    if (NIL_P(c)) {
+	rb_eof_error();
+    }
+    return c;
+}
+
+/*
+ *  call-seq:
+ *     ios.getbyte   => fixnum or nil
+ *
+ *  Gets the next 8-bit byte (0..255) from <em>ios</em>. Returns
+ *  <code>nil</code> if called at end of file.
+ *
+ *     f = File.new("testfile")
+ *     f.getbyte   #=> 84
+ *     f.getbyte   #=> 104
+ */
+
+VALUE
+rb_io_getbyte(VALUE io)
+{
+    rb_io_t *fptr;
+    int c;
+
+    GetOpenFile(io, fptr);
+    rb_io_check_readable(fptr);
+    READ_CHECK(fptr);
+    if (fptr->fd == 0 && (fptr->mode & FMODE_TTY) && TYPE(rb_stdout) == T_FILE) {
+        rb_io_t *ofp;
+        GetOpenFile(rb_stdout, ofp);
+        if (ofp->mode & FMODE_TTY) {
+            rb_io_flush(rb_stdout);
+        }
+    }
+    if (io_fillbuf(fptr) < 0) {
+	return Qnil;
+    }
+    fptr->rbuf_off++;
+    fptr->rbuf_len--;
+    c = (unsigned char)fptr->rbuf[fptr->rbuf_off-1];
+    return INT2FIX(c & 0xff);
+}
+
+/*
+ *  call-seq:
+ *     ios.readbyte   => fixnum
+ *
+ *  Reads a character as with <code>IO#getc</code>, but raises an
+ *  <code>EOFError</code> on end of file.
+ */
+
+static VALUE
+rb_io_readbyte(VALUE io)
+{
+    VALUE c = rb_io_getbyte(io);
 
     if (NIL_P(c)) {
 	rb_eof_error();
@@ -2173,25 +2228,24 @@ rb_io_readchar(VALUE io)
 VALUE
 rb_io_ungetc(VALUE io, VALUE c)
 {
+    rb_encoding *enc;
     rb_io_t *fptr;
-    int cc;
 
     GetOpenFile(io, fptr);
     rb_io_check_readable(fptr);
     if (NIL_P(c)) return Qnil;
+    enc = rb_enc_get(io);
     if (FIXNUM_P(c)) {
-	cc = FIX2INT(c);
+	int cc = FIX2INT(c);
+	char buf[16];
+
+	rb_enc_mbcput(cc, buf, enc);
+	c = rb_str_new(buf, rb_enc_codelen(cc, enc));
     }
     else {
 	SafeStringValue(c);
-	if (RSTRING_LEN(c) > 1) {
-	    rb_warn("IO#ungetc pushes back only one byte");
-	}
-	cc = (unsigned char)RSTRING_PTR(c)[0];
-    }
-    if (io_ungetc(cc, fptr) == EOF && cc != EOF) {
-	rb_raise(rb_eIOError, "ungetc failed");
     }
+    io_ungetc(c, fptr);
     return Qnil;
 }
 
@@ -5465,7 +5519,29 @@ argf_getc(void)
 	ch = rb_funcall3(current_file, rb_intern("getc"), 0, 0);
     }
     else {
-	ch = rb_io_getc_m(current_file);
+	ch = rb_io_getc(current_file);
+    }
+    if (NIL_P(ch) && next_p != -1) {
+	argf_close(current_file);
+	next_p = 1;
+	goto retry;
+    }
+
+    return ch;
+}
+
+static VALUE
+argf_getbyte(void)
+{
+    VALUE ch;
+
+  retry:
+    if (!next_argv()) return Qnil;
+    if (TYPE(current_file) != T_FILE) {
+	ch = rb_funcall3(current_file, rb_intern("getbyte"), 0, 0);
+    }
+    else {
+	ch = rb_io_getbyte(current_file);
     }
     if (NIL_P(ch) && next_p != -1) {
 	argf_close(current_file);
@@ -5478,11 +5554,33 @@ argf_getc(void)
 
 static VALUE
 argf_readchar(void)
+{
+    VALUE ch;
+
+  retry:
+    if (!next_argv()) return Qnil;
+    if (TYPE(current_file) != T_FILE) {
+	ch = rb_funcall3(current_file, rb_intern("getc"), 0, 0);
+    }
+    else {
+	ch = rb_io_getc(current_file);
+    }
+    if (NIL_P(ch) && next_p != -1) {
+	argf_close(current_file);
+	next_p = 1;
+	goto retry;
+    }
+
+    return ch;
+}
+
+static VALUE
+argf_readbyte(void)
 {
     VALUE c;
 
     NEXT_ARGF_FORWARD(0, 0);
-    c = argf_getc();
+    c = argf_getbyte();
     if (NIL_P(c)) {
 	rb_eof_error();
     }
@@ -5780,8 +5878,10 @@ Init_IO(void)
     rb_define_method(rb_cIO, "write", io_write, 1);
     rb_define_method(rb_cIO, "gets",  rb_io_gets_m, -1);
     rb_define_method(rb_cIO, "readline",  rb_io_readline, -1);
-    rb_define_method(rb_cIO, "getc",  rb_io_getc_m, 0);
+    rb_define_method(rb_cIO, "getc",  rb_io_getc, 0);
+    rb_define_method(rb_cIO, "getbyte",  rb_io_getbyte, 0);
     rb_define_method(rb_cIO, "readchar",  rb_io_readchar, 0);
+    rb_define_method(rb_cIO, "readbyte",  rb_io_readbyte, 0);
     rb_define_method(rb_cIO, "ungetc",rb_io_ungetc, 1);
     rb_define_method(rb_cIO, "<<",    rb_io_addstr, 1);
     rb_define_method(rb_cIO, "flush", rb_io_flush, 0);
@@ -5851,7 +5951,9 @@ Init_IO(void)
     rb_define_singleton_method(argf, "gets", rb_f_gets, -1);
     rb_define_singleton_method(argf, "readline", rb_f_readline, -1);
     rb_define_singleton_method(argf, "getc", argf_getc, 0);
+    rb_define_singleton_method(argf, "getbyte", argf_getbyte, 0);
     rb_define_singleton_method(argf, "readchar", argf_readchar, 0);
+    rb_define_singleton_method(argf, "readbyte", argf_readbyte, 0);
     rb_define_singleton_method(argf, "tell", argf_tell, 0);
     rb_define_singleton_method(argf, "seek", argf_seek_m, -1);
     rb_define_singleton_method(argf, "rewind", argf_rewind, 0);

main.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Aug 19 13:19:58 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

marshal.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Apr 27 16:30:01 JST 1995
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
@@ -1105,7 +1105,7 @@ r_object0(struct load_arg *arg, int *ivp, VALUE extmod)
 	{
 	    volatile VALUE str = r_bytes(arg);
 	    int options = r_byte(arg);
-	    v = r_entry(rb_reg_new(RSTRING_PTR(str), RSTRING_LEN(str), options), arg);
+	    v = r_entry(rb_reg_new(str, options), arg);
 	}
 	break;
 

math.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Jan 25 14:12:56 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

numeric.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Aug 13 18:33:09 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

object.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Jul 15 12:01:24 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 

pack.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Feb 10 15:17:05 JST 1994
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

parse.y

@@ -6,7 +6,7 @@
   $Date$
   created at: Fri May 28 18:02:42 JST 1993
 
-  Copyright (C) 1993-2004 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
@@ -20,6 +20,8 @@
 #include "ruby/intern.h"
 #include "ruby/node.h"
 #include "ruby/st.h"
+#include "ruby/encoding.h"
+#include "regenc.h"
 #include <stdio.h>
 #include <errno.h>
 #include <ctype.h>
@@ -255,8 +257,13 @@ struct parser_params {
     VALUE parsing_thread;
     int toplevel_p;
 #endif
+
+    rb_encoding *enc;
 };
 
+#define STR_NEW(p,n) rb_enc_str_new((p),(n),parser->enc)
+#define STR_NEW2(p) rb_enc_str_new((p),strlen(p),parser->enc)
+
 #ifdef YYMALLOC
 void *rb_parser_malloc(struct parser_params *, size_t);
 void *rb_parser_realloc(struct parser_params *, void *, size_t);
@@ -3555,7 +3562,7 @@ strings		: string
 		    /*%%%*/
 			NODE *node = $1;
 			if (!node) {
-			    node = NEW_STR(rb_str_new(0, 0));
+			    node = NEW_STR(STR_NEW(0, 0));
 			}
 			else {
 			    node = evstr2dstr(node);
@@ -3594,7 +3601,7 @@ xstring		: tXSTRING_BEG xstring_contents tSTRING_END
 		    /*%%%*/
 			NODE *node = $2;
 			if (!node) {
-			    node = NEW_XSTR(rb_str_new(0, 0));
+			    node = NEW_XSTR(STR_NEW(0, 0));
 			}
 			else {
 			    switch (nd_type(node)) {
@@ -3605,7 +3612,7 @@ xstring		: tXSTRING_BEG xstring_contents tSTRING_END
 				nd_set_type(node, NODE_DXSTR);
 				break;
 			      default:
-				node = NEW_NODE(NODE_DXSTR, rb_str_new(0, 0), 1, NEW_LIST(node));
+				node = NEW_NODE(NODE_DXSTR, STR_NEW(0, 0), 1, NEW_LIST(node));
 				break;
 			    }
 			}
@@ -3622,20 +3629,18 @@ regexp		: tREGEXP_BEG xstring_contents tREGEXP_END
 			int options = $3;
 			NODE *node = $2;
 			if (!node) {
-			    node = NEW_LIT(reg_compile("", 0, options));
+			    node = NEW_LIT(rb_reg_compile(0, options & ~RE_OPTION_ONCE));
 			}
 			else switch (nd_type(node)) {
 			  case NODE_STR:
 			    {
 				VALUE src = node->nd_lit;
 				nd_set_type(node, NODE_LIT);
-				node->nd_lit = reg_compile(RSTRING_PTR(src),
+				node->nd_lit = rb_reg_compile(src, options&~RE_OPTION_ONCE);
-							   RSTRING_LEN(src),
-							   options);
 			    }
 			    break;
 			  default:
-			    node = NEW_NODE(NODE_DSTR, rb_str_new(0, 0), 1, NEW_LIST(node));
+			    node = NEW_NODE(NODE_DSTR, STR_NEW(0, 0), 1, NEW_LIST(node));
 			  case NODE_DSTR:
 			    if (options & RE_OPTION_ONCE) {
 				nd_set_type(node, NODE_DREGX_ONCE);
@@ -3880,7 +3885,7 @@ dsym		: tSYMBEG xstring_contents tSTRING_END
 				nd_set_type($$, NODE_LIT);
 				break;
 			      default:
-				$$ = NEW_NODE(NODE_DSYM, rb_str_new(0, 0), 1, NEW_LIST($$));
+				$$ = NEW_NODE(NODE_DSYM, STR_NEW(0, 0), 1, NEW_LIST($$));
 				break;
 			    }
 			}
@@ -4518,7 +4523,7 @@ ripper_dispatch_scan_event(struct parser_params *parser, int t)
 
     if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp");
     if (lex_p == parser->tokp) return;
-    str = rb_str_new(parser->tokp, lex_p - parser->tokp);
+    str = STR_NEW(parser->tokp, lex_p - parser->tokp);
     yylval.val = ripper_dispatch1(parser, ripper_token2eventid(t), str);
     ripper_flush(parser);
 }
@@ -4552,7 +4557,11 @@ ripper_dispatch_delayed_token(struct parser_params *parser, int t)
 /* As in Harbison and Steele.  */
 # define SIGN_EXTEND_CHAR(c) ((((unsigned char)(c)) ^ 128) - 128)
 #endif
-#define is_identchar(c) (SIGN_EXTEND_CHAR(c)!=-1&&(ISALNUM(c) || (c) == '_' || ismbchar(c)))
+
+#define parser_mbclen()  mbclen((lex_p-1),parser->enc)
+#define is_identchar(p, enc) (rb_enc_isalnum(*p, enc) || (*p) == '_' || ismbchar(p, enc))
+#define parser_ismbchar() ismbchar((lex_p-1), parser->enc)
+#define parser_is_identchar() (!parser->eofp && is_identchar((lex_p-1),parser->enc))
 
 static int
 parser_yyerror(struct parser_params *parser, const char *msg)
@@ -4596,7 +4605,7 @@ parser_yyerror(struct parser_params *parser, const char *msg)
 	rb_compile_error_append("%s", buf);
     }
 #else
-    dispatch1(parse_error, rb_str_new2(msg));
+    dispatch1(parse_error, STR_NEW2(msg));
 #endif /* !RIPPER */
     return 0;
 }
@@ -4634,7 +4643,7 @@ yycompile(struct parser_params *parser, const char *f, int line)
     if (!compile_for_eval && rb_safe_level() == 0) {
 	ruby_debug_lines = ruby_suppress_tracing(debug_lines, (VALUE)f);
 	if (ruby_debug_lines && line > 1) {
-	    VALUE str = rb_str_new(0,0);
+	    VALUE str = STR_NEW(0,0);
 	    n = line - 1;
 	    do {
 		rb_ary_push(ruby_debug_lines, str);
@@ -4660,7 +4669,15 @@ yycompile(struct parser_params *parser, const char *f, int line)
 	tree = NEW_NIL();
     }
     if (ruby_eval_tree_begin) {
-	tree->nd_body = NEW_PRELUDE(ruby_eval_tree_begin, tree->nd_body);
+	NODE *scope = ruby_eval_tree;
+
+        if (scope) {
+	    scope->nd_body = NEW_PRELUDE(ruby_eval_tree_begin, scope->nd_body);
+	}
+	return scope;
+    }
+    else {
+	return ruby_eval_tree;
     }
     return tree;
 }
@@ -4682,7 +4699,7 @@ lex_get_str(struct parser_params *parser, VALUE s)
 	if (*end++ == '\n') break;
     }
     lex_gets_ptr = end - RSTRING_PTR(s);
-    return rb_str_new(beg, end - beg);
+    return STR_NEW(beg, end - beg);
 }
 
 static VALUE
@@ -5173,8 +5190,8 @@ parser_tokadd_string(struct parser_params *parser,
 		}
 	    }
 	}
-	else if (ismbchar(uc)) {
+	else if (parser_ismbchar()) {
-	    int i, len = mbclen(uc)-1;
+	    int i, len = parser_mbclen()-1;
 
 	    for (i = 0; i < len; i++) {
 		tokadd(c);
@@ -5252,7 +5269,7 @@ parser_parse_string(struct parser_params *parser, NODE *quote)
     }
 
     tokfix();
-    set_yylval_str(rb_str_new(tok(), toklen()));
+    set_yylval_str(STR_NEW(tok(), toklen()));
     return tSTRING_CONTENT;
 }
 
@@ -5278,8 +5295,7 @@ parser_heredoc_identifier(struct parser_params *parser)
 	tokadd(func);
 	term = c;
 	while ((c = nextc()) != -1 && c != term) {
-            uc = (unsigned int)c;
+	    len = parser_mbclen();
-	    len = mbclen(uc);
 	    do {tokadd(c);} while (--len > 0 && (c = nextc()) != -1);
 	}
 	if (c == -1) {
@@ -5289,8 +5305,7 @@ parser_heredoc_identifier(struct parser_params *parser)
 	break;
 
       default:
-        uc = (unsigned int)c;
+	if (!parser_is_identchar()) {
-	if (!is_identchar(uc)) {
 	    pushback(c);
 	    if (func & STR_FUNC_INDENT) {
 		pushback('-');
@@ -5301,11 +5316,9 @@ parser_heredoc_identifier(struct parser_params *parser)
 	term = '"';
 	tokadd(func |= str_dquote);
 	do {
-            uc = (unsigned int)c;
+	    len = parser_mbclen();
-	    len = mbclen(uc);
 	    do {tokadd(c);} while (--len > 0 && (c = nextc()) != -1);
-	} while ((c = nextc()) != -1 &&
+	} while ((c = nextc()) != -1 && parser_is_identchar());
-		 (uc = (unsigned char)c, is_identchar(uc)));
 	pushback(c);
 	break;
     }
@@ -5317,7 +5330,7 @@ parser_heredoc_identifier(struct parser_params *parser)
     len = lex_p - lex_pbeg;
     lex_goto_eol(parser);
     lex_strterm = rb_node_newnode(NODE_HEREDOC,
-				  rb_str_new(tok(), toklen()),	/* nd_lit */
+				  STR_NEW(tok(), toklen()),	/* nd_lit */
 				  len,				/* nd_nth */
 				  lex_lastline);		/* nd_orig */
     nd_set_line(lex_strterm, ruby_sourceline);
@@ -5410,7 +5423,7 @@ parser_here_document(struct parser_params *parser, NODE *here)
 	    if (str)
 		rb_str_cat(str, p, pend - p);
 	    else
-		str = rb_str_new(p, pend - p);
+		str = STR_NEW(p, pend - p);
 	    if (pend < lex_pend) rb_str_cat(str, "\n", 1);
 	    lex_goto_eol(parser);
 	    if (nextc() == -1) {
@@ -5436,13 +5449,13 @@ parser_here_document(struct parser_params *parser, NODE *here)
 	    pushback(c);
 	    if ((c = tokadd_string(func, '\n', 0, NULL)) == -1) goto error;
 	    if (c != '\n') {
-                set_yylval_str(rb_str_new(tok(), toklen()));
+                set_yylval_str(STR_NEW(tok(), toklen()));
 		return tSTRING_CONTENT;
 	    }
 	    tokadd(nextc());
 	    if ((c = nextc()) == -1) goto error;
 	} while (!whole_match_p(eos, len, indent));
-	str = rb_str_new(tok(), toklen());
+	str = STR_NEW(tok(), toklen());
     }
     heredoc_restore(lex_strterm);
     lex_strterm = NEW_STRTERM(-1, 0, 0);
@@ -5487,6 +5500,7 @@ pragma_encoding(struct parser_params *parser, const char *name, const char *val)
     if (parser && parser->line_count != (parser->has_shebang ? 2 : 1))
 	return;
     rb_set_kcode(val);
+    parser->enc = rb_enc_find(val);
 }
 
 struct pragma {
@@ -5540,7 +5554,7 @@ parser_pragma(struct parser_params *parser, const char *str, int len)
 #define str_copy(_s, _p, _n) ((_s) \
 	? (rb_str_resize((_s), (_n)), \
 	   MEMCPY(RSTRING_PTR(_s), (_p), char, (_n)), (_s)) \
-	: ((_s) = rb_str_new((_p), (_n))))
+	: ((_s) = STR_NEW((_p), (_n))))
 
     if (len <= 7) return Qfalse;
     if (!(beg = pragma_marker(str, len))) return Qfalse;
@@ -5934,8 +5948,7 @@ parser_yylex(struct parser_params *parser)
 	    compile_error(PARSER_ARG "incomplete character syntax");
 	    return 0;
 	}
-        uc = (unsigned char)c;
+	if (rb_enc_isspace(c, parser->enc)){
-	if (ISSPACE(c)){
 	    if (!IS_ARG()){
 		int c2 = 0;
 		switch (c) {
@@ -5968,8 +5981,8 @@ parser_yylex(struct parser_params *parser)
 	    return '?';
 	}
 	newtok();
-	if (ismbchar(uc)) {
+	if (parser_ismbchar()) {
-	    int i, len = mbclen(uc)-1;
+	    int i, len = parser_mbclen()-1;
 
 	    tokadd(c);
 	    for (i = 0; i < len; i++) {
@@ -5977,7 +5990,8 @@ parser_yylex(struct parser_params *parser)
 		tokadd(c);
 	    }
 	}
-	else if ((ISALNUM(c) || c == '_') && lex_p < lex_pend && is_identchar(*lex_p)) {
+	else if ((rb_enc_isalnum(c, parser->enc) || c == '_') &&
+		 lex_p < lex_pend && is_identchar(lex_p, parser->enc)) {
 	    goto ternary;
 	}
 	else if (c == '\\') {
@@ -5988,7 +6002,7 @@ parser_yylex(struct parser_params *parser)
 	    tokadd(c);
 	}
 	tokfix();
-	set_yylval_str(rb_str_new(tok(), toklen()));
+	set_yylval_str(STR_NEW(tok(), toklen()));
 	lex_state = EXPR_ENDARG;
 	return tCHAR;
 
@@ -6544,8 +6558,7 @@ parser_yylex(struct parser_params *parser)
 	    }
 	    else {
 		term = nextc();
-                uc = (unsigned char)c;
+		if (rb_enc_isalnum(term, parser->enc) || parser_ismbchar()) {
-		if (ISALNUM(term) || ismbchar(uc)) {
 		    yyerror("unknown type of %string");
 		    return 0;
 		}
@@ -6625,8 +6638,7 @@ parser_yylex(struct parser_params *parser)
 	switch (c) {
 	  case '_':		/* $_: last read line string */
 	    c = nextc();
-            uc = (unsigned char)c;
+	    if (parser_is_identchar()) {
-	    if (is_identchar(uc)) {
 		tokadd('$');
 		tokadd('_');
 		break;
@@ -6660,8 +6672,7 @@ parser_yylex(struct parser_params *parser)
 	    tokadd('$');
 	    tokadd(c);
 	    c = nextc();
-	    uc = (unsigned char)c;
+	    if (parser_is_identchar()) {
-	    if (is_identchar(uc)) {
 	    tokadd(c);
 	    }
 	    else {
@@ -6703,8 +6714,7 @@ parser_yylex(struct parser_params *parser)
 	    return tNTH_REF;
 
 	  default:
-            uc = (unsigned char)c;
+	    if (!parser_is_identchar()) {
-	    if (!is_identchar(uc)) {
 		pushback(c);
 		return '$';
 	    }
@@ -6730,8 +6740,7 @@ parser_yylex(struct parser_params *parser)
 	    }
 	    return 0;
 	}
-        uc = (unsigned char)c;
+	if (!parser_is_identchar()) {
-	if (!is_identchar(uc)) {
 	    pushback(c);
 	    return '@';
 	}
@@ -6753,9 +6762,8 @@ parser_yylex(struct parser_params *parser)
 	break;
 
       default:
-	uc = (unsigned char)c;
+	if (!parser_is_identchar()) {
-	if (!is_identchar(uc)) {
+	    rb_compile_error(PARSER_ARG  "Invalid char `\\%03o' in expression", c);
-	    compile_error(PARSER_ARG "Invalid char `\\%03o' in expression", c);
 	    goto retry;
 	}
 
@@ -6763,21 +6771,18 @@ parser_yylex(struct parser_params *parser)
 	break;
     }
 
-    uc = (unsigned char)c;
     do {
+        int i, len;
 	tokadd(c);
-	if (ismbchar(uc)) {
-	    int i, len = mbclen(uc)-1;
 
-	    for (i = 0; i < len; i++) {
+	len = parser_mbclen()-1;
-		c = nextc();
+        for (i = 0; i < len; i++) {
-		tokadd(c);
+	    c = nextc();
-	    }
+	    tokadd(c);
 	}
 	c = nextc();
-        uc = (unsigned char)c;
+    } while (parser_is_identchar());
-    } while (is_identchar(uc));
+    if ((c == '!' || c == '?') && !peek('=')) {
-    if ((c == '!' || c == '?') && is_identchar(tok()[0]) && !peek('=')) {
 	tokadd(c);
     }
     else {
@@ -7214,7 +7219,7 @@ gettable_gen(struct parser_params *parser, ID id)
 	return NEW_FALSE();
     }
     else if (id == keyword__FILE__) {
-	return NEW_STR(rb_str_new2(ruby_sourcefile));
+	return NEW_STR(STR_NEW2(ruby_sourcefile));
     }
     else if (id == keyword__LINE__) {
 	return NEW_LIT(INT2FIX(ruby_sourceline));
@@ -8115,8 +8120,7 @@ dvar_curr_gen(struct parser_params *parser, ID id)
 static VALUE
 reg_compile_gen(struct parser_params* parser, const char *ptr, long len, int options)
 {
-    VALUE rb_reg_compile(const char *, long, int);
+    VALUE re = rb_reg_compile(STR_NEW(ptr, len), (options) & ~RE_OPTION_ONCE);
-    VALUE re = rb_reg_compile(ptr, len, (options) & ~RE_OPTION_ONCE);
 
     if (NIL_P(re)) {
 	RB_GC_GUARD(re) = rb_obj_as_string(rb_errinfo());
@@ -8316,7 +8320,7 @@ internal_id_gen(struct parser_params *parser)
 }
 
 static int
-is_special_global_name(const char *m)
+is_special_global_name(const char *m, rb_encoding *enc)
 {
     switch (*m) {
       case '~': case '*': case '$': case '?': case '!': case '@':
@@ -8328,11 +8332,11 @@ is_special_global_name(const char *m)
 	break;
       case '-':
 	++m;
-	if (is_identchar(*m)) m += mbclen(*m);
+	if (is_identchar(m, enc)) m += rb_enc_mbclen(m, enc);
 	break;
       default:
-	if (!ISDIGIT(*m)) return 0;
+	if (!rb_enc_isdigit(*m, enc)) return 0;
-	do ++m; while (ISDIGIT(*m));
+	do ++m; while (rb_enc_isdigit(*m, enc));
     }
     return !*m;
 }
@@ -8342,6 +8346,7 @@ rb_symname_p(const char *name)
 {
     const char *m = name;
     int localid = Qfalse;
+    rb_encoding *enc = rb_enc_from_index(0);
 
     if (!m) return Qfalse;
     switch (*m) {
@@ -8349,7 +8354,7 @@ rb_symname_p(const char *name)
 	return Qfalse;
 
       case '$':
-	if (is_special_global_name(++m)) return Qtrue;
+	if (is_special_global_name(++m, enc)) return Qtrue;
 	goto id;
 
       case '@':
@@ -8396,10 +8401,10 @@ rb_symname_p(const char *name)
 	break;
 
       default:
-	localid = !ISUPPER(*m);
+	localid = !rb_enc_isupper(*m, enc);
       id:
-	if (*m != '_' && !ISALPHA(*m) && !ismbchar(*m)) return Qfalse;
+	if (*m != '_' && !rb_enc_isalpha(*m, enc) && !ismbchar(m, enc)) return Qfalse;
-	while (is_identchar(*m)) m += mbclen(*m);
+	while (is_identchar(m, enc)) m += rb_enc_mbclen(m, enc);
 	if (localid) {
 	    switch (*m) {
 	      case '!': case '?': case '=': ++m;
@@ -8411,7 +8416,7 @@ rb_symname_p(const char *name)
 }
 
 ID
-rb_intern2(const char *name, long len)
+rb_intern3(const char *name, long len, rb_encoding *enc)
 {
     const char *m = name;
     VALUE str;
@@ -8429,13 +8434,13 @@ rb_intern2(const char *name, long len)
 
     last = len-1;
     id = 0;
-    switch (*name) {
+    switch (*m) {
       case '$':
 	id |= ID_GLOBAL;
-	if (is_special_global_name(++m)) goto new_id;
+	if (is_special_global_name(++m, enc)) goto new_id;
 	break;
       case '@':
-	if (name[1] == '@') {
+	if (m[1] == '@') {
 	    m++;
 	    id |= ID_CLASS;
 	}
@@ -8445,20 +8450,21 @@ rb_intern2(const char *name, long len)
 	m++;
 	break;
       default:
-	if (name[0] != '_' && ISASCII(name[0]) && !ISALNUM(name[0])) {
+	if (m[0] != '_' && rb_enc_isascii((unsigned char)m[0], enc)
+	    && !rb_enc_isalnum(m[0], enc)) {
 	    /* operators */
 	    int i;
 
 	    for (i=0; op_tbl[i].token; i++) {
-		if (*op_tbl[i].name == *name &&
+		if (*op_tbl[i].name == *m &&
-		    strcmp(op_tbl[i].name, name) == 0) {
+		    strcmp(op_tbl[i].name, m) == 0) {
 		    id = op_tbl[i].token;
 		    goto id_register;
 		}
 	    }
 	}
 
-	if (name[last] == '=') {
+	if (m[last] == '=') {
 	    /* attribute assignment */
 	    id = rb_intern2(name, last);
 	    if (id > tLAST_TOKEN && !is_attrset_id(id)) {
@@ -8467,7 +8473,7 @@ rb_intern2(const char *name, long len)
 	    }
 	    id = ID_ATTRSET;
 	}
-	else if (ISUPPER(name[0])) {
+	else if (rb_enc_isupper(m[0], enc)) {
 	    id = ID_CONST;
         }
 	else {
@@ -8475,9 +8481,9 @@ rb_intern2(const char *name, long len)
 	}
 	break;
     }
-    if (!ISDIGIT(*m)) {
+    if (!rb_enc_isdigit(*m, enc)) {
-	while (m <= name + last && is_identchar(*m)) {
+	while (m <= name + last && is_identchar(m, enc)) {
-	    m += mbclen(*m);
+	    m += rb_enc_mbclen(m, enc);
 	}
     }
     if (m - name < len) id = ID_JUNK;
@@ -8491,12 +8497,24 @@ rb_intern2(const char *name, long len)
     return id;
 }
 
+ID
+rb_intern2(const char *name, long len)
+{
+    return rb_intern3(name, len, rb_enc_from_index(0));
+}
+
 ID
 rb_intern(const char *name)
 {
     return rb_intern2(name, strlen(name));
 }
 
+ID
+rb_intern_str(VALUE str)
+{
+    return rb_intern3(RSTRING_PTR(str), RSTRING_LEN(str), rb_enc_get(str));
+}
+
 VALUE
 rb_id2str(ID id)
 {
@@ -8662,6 +8680,7 @@ parser_initialize(struct parser_params *parser)
 #ifdef YYMALLOC
     parser->heap = NULL;
 #endif
+    parser->enc = rb_enc_from_index(0);
 }
 
 extern void rb_mark_source_filename(char *);
@@ -9013,27 +9032,27 @@ ripper_compile_error(struct parser_params *parser, const char *fmt, ...)
 static void
 ripper_warn0(struct parser_params *parser, const char *fmt)
 {
-    rb_funcall(parser->value, rb_intern("warn"), 1, rb_str_new2(fmt));
+    rb_funcall(parser->value, rb_intern("warn"), 1, STR_NEW2(fmt));
 }
 
 static void
 ripper_warnI(struct parser_params *parser, const char *fmt, int a)
 {
     rb_funcall(parser->value, rb_intern("warn"), 2,
-               rb_str_new2(fmt), INT2NUM(a));
+               STR_NEW2(fmt), INT2NUM(a));
 }
 
 static void
 ripper_warnS(struct parser_params *parser, const char *fmt, const char *str)
 {
     rb_funcall(parser->value, rb_intern("warn"), 2,
-    	       rb_str_new2(fmt), rb_str_new2(str));
+    	       STR_NEW2(fmt), STR_NEW2(str));
 }
 
 static void
 ripper_warning0(struct parser_params *parser, const char *fmt)
 {
-    rb_funcall(parser->value, rb_intern("warning"), 1, rb_str_new2(fmt));
+    rb_funcall(parser->value, rb_intern("warning"), 1, STR_NEW2(fmt));
 }
 
 #if 0				/* unused in ripper right now */
@@ -9041,7 +9060,7 @@ static void
 ripper_warningS(struct parser_params *parser, const char *fmt, const char *str)
 {
     rb_funcall(parser->value, rb_intern("warning"), 2,
-    	       rb_str_new2(fmt), rb_str_new2(str));
+    	       STR_NEW2(fmt), STR_NEW2(str));
 }
 #endif
 
@@ -9094,7 +9113,7 @@ ripper_initialize(int argc, VALUE *argv, VALUE self)
     parser->parser_lex_input = src;
     parser->eofp = Qfalse;
     if (NIL_P(fname)) {
-        fname = rb_str_new2("(ripper)");
+        fname = STR_NEW2("(ripper)");
     }
     else {
         StringValue(fname);

prec.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Jan 26 02:40:41 2000
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 

process.c

@@ -6,7 +6,7 @@
   $Date$
   created at: Tue Aug 10 14:30:50 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan