* parse.y [ripper]: on__scan event removed.

* parse.y [ripper]: event name is changed: on__XXX -> on_XXX.
* ext/ripper/eventids2.c: ditto.
* ext/ripper/ripper.rb.in: ditto.
* ext/ripper/lib/ripper.rb: sync with ripper.rb.in.
* ext/ripper/lib/ripper/tokenizer: ditto.
* ext/ripper/lib/ripper/filter: new file.
* sample/ripper/colorize.rb: new file.
* sample/ripper/strip-comment.rb: new file.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6952 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,23 @@
+Wed Sep 22 14:21:54 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* parse.y [ripper]: on__scan event removed.
+
+	* parse.y [ripper]: event name is changed: on__XXX -> on_XXX.
+
+	* ext/ripper/eventids2.c: ditto.
+
+	* ext/ripper/ripper.rb.in: ditto.
+
+	* ext/ripper/lib/ripper.rb: sync with ripper.rb.in.
+
+	* ext/ripper/lib/ripper/tokenizer: ditto.
+
+	* ext/ripper/lib/ripper/filter: new file.
+
+	* sample/ripper/colorize.rb: new file.
+
+	* sample/ripper/strip-comment.rb: new file.
+
 Wed Sep 22 13:50:49 2004  Yukihiro Matsumoto  <matz@ruby-lang.org>
 
 	* hash.c (hash_alloc): was using tbl pointer without

MANIFEST

@@ -735,6 +735,8 @@ sample/rcs.awk
 sample/rcs.dat
 sample/rcs.rb
 sample/regx.rb
+sample/ripper/colorize.rb
+sample/ripper/strip-comment.rb
 sample/rss/list_description.rb
 sample/rss/rss_recent.rb
 sample/rss/tdiary_plugin/rss-recent.rb

ext/ripper/MANIFEST

@@ -6,6 +6,7 @@ eventids2.c
 extconf.rb
 ripper.rb.in
 lib/ripper.rb
+lib/ripper/filter.rb
 lib/ripper/tokenizer.rb
 test/check-event-arity.rb
 test/check-event-coverage.sh

ext/ripper/eventids2.c

@@ -9,8 +9,6 @@
 #define k__END__         (tLAST_TOKEN + 9)
 #define tCHAR            (tLAST_TOKEN + 10)
 
-static ID ripper_id_scan;
-
 static ID ripper_id_backref;
 static ID ripper_id_backtick;
 static ID ripper_id_comma;
@@ -59,52 +57,50 @@ static ID ripper_id_CHAR;
 static void
 ripper_init_eventids2()
 {
-    ripper_id_scan = rb_intern("on__scan");
+    ripper_id_backref = rb_intern("on_backref");
+    ripper_id_backtick = rb_intern("on_backtick");
+    ripper_id_comma = rb_intern("on_comma");
+    ripper_id_const = rb_intern("on_const");
+    ripper_id_cvar = rb_intern("on_cvar");
+    ripper_id_embexpr_beg = rb_intern("on_embexpr_beg");
+    ripper_id_embexpr_end = rb_intern("on_embexpr_end");
+    ripper_id_embvar = rb_intern("on_embvar");
+    ripper_id_float = rb_intern("on_float");
+    ripper_id_gvar = rb_intern("on_gvar");
+    ripper_id_ident = rb_intern("on_ident");
+    ripper_id_int = rb_intern("on_int");
+    ripper_id_ivar = rb_intern("on_ivar");
+    ripper_id_kw = rb_intern("on_kw");
+    ripper_id_lbrace = rb_intern("on_lbrace");
+    ripper_id_lbracket = rb_intern("on_lbracket");
+    ripper_id_lparen = rb_intern("on_lparen");
+    ripper_id_nl = rb_intern("on_nl");
+    ripper_id_op = rb_intern("on_op");
+    ripper_id_period = rb_intern("on_period");
+    ripper_id_rbrace = rb_intern("on_rbrace");
+    ripper_id_rbracket = rb_intern("on_rbracket");
+    ripper_id_rparen = rb_intern("on_rparen");
+    ripper_id_semicolon = rb_intern("on_semicolon");
+    ripper_id_symbeg = rb_intern("on_symbeg");
+    ripper_id_tstring_beg = rb_intern("on_tstring_beg");
+    ripper_id_tstring_content = rb_intern("on_tstring_content");
+    ripper_id_tstring_end = rb_intern("on_tstring_end");
+    ripper_id_words_beg = rb_intern("on_words_beg");
+    ripper_id_qwords_beg = rb_intern("on_qwords_beg");
+    ripper_id_words_sep = rb_intern("on_words_sep");
+    ripper_id_regexp_beg = rb_intern("on_regexp_beg");
+    ripper_id_regexp_end = rb_intern("on_regexp_end");
 
-    ripper_id_backref = rb_intern("on__backref");
+    ripper_id_ignored_nl = rb_intern("on_ignored_nl");
-    ripper_id_backtick = rb_intern("on__backtick");
+    ripper_id_comment = rb_intern("on_comment");
-    ripper_id_comma = rb_intern("on__comma");
+    ripper_id_embdoc_beg = rb_intern("on_embdoc_beg");
-    ripper_id_const = rb_intern("on__const");
+    ripper_id_embdoc = rb_intern("on_embdoc");
-    ripper_id_cvar = rb_intern("on__cvar");
+    ripper_id_embdoc_end = rb_intern("on_embdoc_end");
-    ripper_id_embexpr_beg = rb_intern("on__embexpr_beg");
+    ripper_id_sp = rb_intern("on_sp");
-    ripper_id_embexpr_end = rb_intern("on__embexpr_end");
+    ripper_id_heredoc_beg = rb_intern("on_heredoc_beg");
-    ripper_id_embvar = rb_intern("on__embvar");
+    ripper_id_heredoc_end = rb_intern("on_heredoc_end");
-    ripper_id_float = rb_intern("on__float");
+    ripper_id___end__ = rb_intern("on___end__");
-    ripper_id_gvar = rb_intern("on__gvar");
+    ripper_id_CHAR = rb_intern("on_CHAR");
-    ripper_id_ident = rb_intern("on__ident");
-    ripper_id_int = rb_intern("on__int");
-    ripper_id_ivar = rb_intern("on__ivar");
-    ripper_id_kw = rb_intern("on__kw");
-    ripper_id_lbrace = rb_intern("on__lbrace");
-    ripper_id_lbracket = rb_intern("on__lbracket");
-    ripper_id_lparen = rb_intern("on__lparen");
-    ripper_id_nl = rb_intern("on__nl");
-    ripper_id_op = rb_intern("on__op");
-    ripper_id_period = rb_intern("on__period");
-    ripper_id_rbrace = rb_intern("on__rbrace");
-    ripper_id_rbracket = rb_intern("on__rbracket");
-    ripper_id_rparen = rb_intern("on__rparen");
-    ripper_id_semicolon = rb_intern("on__semicolon");
-    ripper_id_symbeg = rb_intern("on__symbeg");
-    ripper_id_tstring_beg = rb_intern("on__tstring_beg");
-    ripper_id_tstring_content = rb_intern("on__tstring_content");
-    ripper_id_tstring_end = rb_intern("on__tstring_end");
-    ripper_id_words_beg = rb_intern("on__words_beg");
-    ripper_id_qwords_beg = rb_intern("on__qwords_beg");
-    ripper_id_words_sep = rb_intern("on__words_sep");
-    ripper_id_regexp_beg = rb_intern("on__regexp_beg");
-    ripper_id_regexp_end = rb_intern("on__regexp_end");
-
-    ripper_id_ignored_nl = rb_intern("on__ignored_nl");
-    ripper_id_comment = rb_intern("on__comment");
-    ripper_id_embdoc_beg = rb_intern("on__embdoc_beg");
-    ripper_id_embdoc = rb_intern("on__embdoc");
-    ripper_id_embdoc_end = rb_intern("on__embdoc_end");
-    ripper_id_sp = rb_intern("on__sp");
-    ripper_id_heredoc_beg = rb_intern("on__heredoc_beg");
-    ripper_id_heredoc_end = rb_intern("on__heredoc_end");
-    ripper_id___end__ = rb_intern("on____end__");
-    ripper_id_CHAR = rb_intern("on__CHAR");
 }
 
 static struct token_assoc {

ext/ripper/lib/ripper.rb

@@ -146,7 +146,6 @@ class Ripper
 
   # This table contains name of scanner events and its arity
   # (arity is always 1 for all scanner events).
-  # on__scan is NOT a scanner event.
   SCANNER_EVENT_TABLE = {
     :CHAR => 1,
     :__end__ => 1,
@@ -196,7 +195,7 @@ class Ripper
   # This array contains name of scanner events.
   SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
 
-  # This table contains name of all ripper events, except on__scan.
+  # This table contains name of all ripper events.
   EVENTS = PARSER_EVENTS + SCANNER_EVENTS
 
   ###                ###
@@ -218,483 +217,483 @@ class Ripper
   # Parser Events
   #
 
-  def on__BEGIN(a)
+  def on_BEGIN(a)
     a
   end
 
-  def on__END(a)
+  def on_END(a)
     a
   end
 
-  def on__alias(a, b)
+  def on_alias(a, b)
     a
   end
 
-  def on__alias_error(a)
+  def on_alias_error(a)
     a
   end
 
-  def on__aref(a, b)
+  def on_aref(a, b)
     a
   end
 
-  def on__aref_field(a, b)
+  def on_aref_field(a, b)
     a
   end
 
-  def on__arg_ambiguous
+  def on_arg_ambiguous
     nil
   end
 
-  def on__arg_paren(a)
+  def on_arg_paren(a)
     a
   end
 
-  def on__arglist_add(a, b)
+  def on_arglist_add(a, b)
     a
   end
 
-  def on__arglist_add_block(a, b)
+  def on_arglist_add_block(a, b)
     a
   end
 
-  def on__arglist_add_star(a, b)
+  def on_arglist_add_star(a, b)
     a
   end
 
-  def on__arglist_new
+  def on_arglist_new
     nil
   end
 
-  def on__arglist_prepend(a, b)
+  def on_arglist_prepend(a, b)
     a
   end
 
-  def on__array(a)
+  def on_array(a)
     a
   end
 
-  def on__assign(a, b)
+  def on_assign(a, b)
     a
   end
 
-  def on__assign_error(a)
+  def on_assign_error(a)
     a
   end
 
-  def on__assoc_new(a, b)
+  def on_assoc_new(a, b)
     a
   end
 
-  def on__assoclist_from_args(a)
+  def on_assoclist_from_args(a)
     a
   end
 
-  def on__bare_assoc_hash(a)
+  def on_bare_assoc_hash(a)
     a
   end
 
-  def on__begin(a)
+  def on_begin(a)
     a
   end
 
-  def on__binary(a, b, c)
+  def on_binary(a, b, c)
     a
   end
 
-  def on__blockvar_add_block(a, b)
+  def on_blockvar_add_block(a, b)
     a
   end
 
-  def on__blockvar_add_star(a, b)
+  def on_blockvar_add_star(a, b)
     a
   end
 
-  def on__blockvar_new(a)
+  def on_blockvar_new(a)
     a
   end
 
-  def on__bodystmt(a, b, c, d)
+  def on_bodystmt(a, b, c, d)
     a
   end
 
-  def on__brace_block(a, b)
+  def on_brace_block(a, b)
     a
   end
 
-  def on__break(a)
+  def on_break(a)
     a
   end
 
-  def on__call(a, b, c)
+  def on_call(a, b, c)
     a
   end
 
-  def on__case(a, b)
+  def on_case(a, b)
     a
   end
 
-  def on__class(a, b, c)
+  def on_class(a, b, c)
     a
   end
 
-  def on__class_name_error(a)
+  def on_class_name_error(a)
     a
   end
 
-  def on__command(a, b)
+  def on_command(a, b)
     a
   end
 
-  def on__command_call(a, b, c, d)
+  def on_command_call(a, b, c, d)
     a
   end
 
-  def on__const_ref(a)
+  def on_const_ref(a)
     a
   end
 
-  def on__constpath_field(a, b)
+  def on_constpath_field(a, b)
     a
   end
 
-  def on__constpath_ref(a, b)
+  def on_constpath_ref(a, b)
     a
   end
 
-  def on__def(a, b, c)
+  def on_def(a, b, c)
     a
   end
 
-  def on__defined(a)
+  def on_defined(a)
     a
   end
 
-  def on__defs(a, b, c, d, e)
+  def on_defs(a, b, c, d, e)
     a
   end
 
-  def on__do_block(a, b)
+  def on_do_block(a, b)
     a
   end
 
-  def on__dot2(a, b)
+  def on_dot2(a, b)
     a
   end
 
-  def on__dot3(a, b)
+  def on_dot3(a, b)
     a
   end
 
-  def on__dyna_symbol(a)
+  def on_dyna_symbol(a)
     a
   end
 
-  def on__else(a)
+  def on_else(a)
     a
   end
 
-  def on__elsif(a, b, c)
+  def on_elsif(a, b, c)
     a
   end
 
-  def on__ensure(a)
+  def on_ensure(a)
     a
   end
 
-  def on__fcall(a)
+  def on_fcall(a)
     a
   end
 
-  def on__field(a, b, c)
+  def on_field(a, b, c)
     a
   end
 
-  def on__for(a, b, c)
+  def on_for(a, b, c)
     a
   end
 
-  def on__hash(a)
+  def on_hash(a)
     a
   end
 
-  def on__if(a, b, c)
+  def on_if(a, b, c)
     a
   end
 
-  def on__if_mod(a, b)
+  def on_if_mod(a, b)
     a
   end
 
-  def on__ifop(a, b, c)
+  def on_ifop(a, b, c)
     a
   end
 
-  def on__iter_block(a, b)
+  def on_iter_block(a, b)
     a
   end
 
-  def on__massign(a, b)
+  def on_massign(a, b)
     a
   end
 
-  def on__method_add_arg(a, b)
+  def on_method_add_arg(a, b)
     a
   end
 
-  def on__mlhs_add(a, b)
+  def on_mlhs_add(a, b)
     a
   end
 
-  def on__mlhs_add_star(a, b)
+  def on_mlhs_add_star(a, b)
     a
   end
 
-  def on__mlhs_new
+  def on_mlhs_new
     nil
   end
 
-  def on__mlhs_paren(a)
+  def on_mlhs_paren(a)
     a
   end
 
-  def on__module(a, b)
+  def on_module(a, b)
     a
   end
 
-  def on__mrhs_add(a, b)
+  def on_mrhs_add(a, b)
     a
   end
 
-  def on__mrhs_add_star(a, b)
+  def on_mrhs_add_star(a, b)
     a
   end
 
-  def on__mrhs_new
+  def on_mrhs_new
     nil
   end
 
-  def on__mrhs_new_from_arglist(a)
+  def on_mrhs_new_from_arglist(a)
     a
   end
 
-  def on__next(a)
+  def on_next(a)
     a
   end
 
-  def on__opassign(a, b, c)
+  def on_opassign(a, b, c)
     a
   end
 
-  def on__param_error(a)
+  def on_param_error(a)
     a
   end
 
-  def on__params(a, b, c, d)
+  def on_params(a, b, c, d)
     a
   end
 
-  def on__paren(a)
+  def on_paren(a)
     a
   end
 
-  def on__parse_error(a)
+  def on_parse_error(a)
     a
   end
 
-  def on__program(a)
+  def on_program(a)
     a
   end
 
-  def on__qwords_add(a, b)
+  def on_qwords_add(a, b)
     a
   end
 
-  def on__qwords_new
+  def on_qwords_new
     nil
   end
 
-  def on__redo
+  def on_redo
     nil
   end
 
-  def on__regexp_literal(a)
+  def on_regexp_literal(a)
     a
   end
 
-  def on__rescue(a, b, c, d)
+  def on_rescue(a, b, c, d)
     a
   end
 
-  def on__rescue_mod(a, b)
+  def on_rescue_mod(a, b)
     a
   end
 
-  def on__restparam(a)
+  def on_restparam(a)
     a
   end
 
-  def on__retry
+  def on_retry
     nil
   end
 
-  def on__return(a)
+  def on_return(a)
     a
   end
 
-  def on__return0
+  def on_return0
     nil
   end
 
-  def on__sclass(a, b)
+  def on_sclass(a, b)
     a
   end
 
-  def on__space(a)
+  def on_space(a)
     a
   end
 
-  def on__stmts_add(a, b)
+  def on_stmts_add(a, b)
     a
   end
 
-  def on__stmts_new
+  def on_stmts_new
     nil
   end
 
-  def on__string_add(a, b)
+  def on_string_add(a, b)
     a
   end
 
-  def on__string_concat(a, b)
+  def on_string_concat(a, b)
     a
   end
 
-  def on__string_content
+  def on_string_content
     nil
   end
 
-  def on__string_dvar(a)
+  def on_string_dvar(a)
     a
   end
 
-  def on__string_embexpr(a)
+  def on_string_embexpr(a)
     a
   end
 
-  def on__string_literal(a)
+  def on_string_literal(a)
     a
   end
 
-  def on__super(a)
+  def on_super(a)
     a
   end
 
-  def on__symbol(a)
+  def on_symbol(a)
     a
   end
 
-  def on__symbol_literal(a)
+  def on_symbol_literal(a)
     a
   end
 
-  def on__topconst_field(a)
+  def on_topconst_field(a)
     a
   end
 
-  def on__topconst_ref(a)
+  def on_topconst_ref(a)
     a
   end
 
-  def on__unary(a, b)
+  def on_unary(a, b)
     a
   end
 
-  def on__undef(a)
+  def on_undef(a)
     a
   end
 
-  def on__unless(a, b, c)
+  def on_unless(a, b, c)
     a
   end
 
-  def on__unless_mod(a, b)
+  def on_unless_mod(a, b)
     a
   end
 
-  def on__until(a, b)
+  def on_until(a, b)
     a
   end
 
-  def on__until_mod(a, b)
+  def on_until_mod(a, b)
     a
   end
 
-  def on__var_alias(a, b)
+  def on_var_alias(a, b)
     a
   end
 
-  def on__var_field(a)
+  def on_var_field(a)
     a
   end
 
-  def on__var_ref(a)
+  def on_var_ref(a)
     a
   end
 
-  def on__void_stmt
+  def on_void_stmt
     nil
   end
 
-  def on__when(a, b, c)
+  def on_when(a, b, c)
     a
   end
 
-  def on__while(a, b)
+  def on_while(a, b)
     a
   end
 
-  def on__while_mod(a, b)
+  def on_while_mod(a, b)
     a
   end
 
-  def on__word_add(a, b)
+  def on_word_add(a, b)
     a
   end
 
-  def on__word_new
+  def on_word_new
     nil
   end
 
-  def on__words_add(a, b)
+  def on_words_add(a, b)
     a
   end
 
-  def on__words_new
+  def on_words_new
     nil
   end
 
-  def on__xstring_add(a, b)
+  def on_xstring_add(a, b)
     a
   end
 
-  def on__xstring_literal(a)
+  def on_xstring_literal(a)
     a
   end
 
-  def on__xstring_new
+  def on_xstring_new
     nil
   end
 
-  def on__yield(a)
+  def on_yield(a)
     a
   end
 
-  def on__yield0
+  def on_yield0
     nil
   end
 
-  def on__zsuper
+  def on_zsuper
     nil
   end
 
@@ -702,178 +701,177 @@ class Ripper
   # Lexer Events
   #
 
-  def on__scan(event, token)
+  def on_CHAR(token)
-  end
-
-  def on__CHAR(token)
     token
   end
 
-  def on____end__(token)
+  def on___end__(token)
     token
   end
 
-  def on__backref(token)
+  def on_backref(token)
     token
   end
 
-  def on__backtick(token)
+  def on_backtick(token)
     token
   end
 
-  def on__comma(token)
+  def on_comma(token)
     token
   end
 
-  def on__comment(token)
+  def on_comment(token)
     token
   end
 
-  def on__const(token)
+  def on_const(token)
     token
   end
 
-  def on__cvar(token)
+  def on_cvar(token)
     token
   end
 
-  def on__embdoc(token)
+  def on_embdoc(token)
     token
   end
 
-  def on__embdoc_beg(token)
+  def on_embdoc_beg(token)
     token
   end
 
-  def on__embdoc_end(token)
+  def on_embdoc_end(token)
     token
   end
 
-  def on__embexpr_beg(token)
+  def on_embexpr_beg(token)
     token
   end
 
-  def on__embexpr_end(token)
+  def on_embexpr_end(token)
     token
   end
 
-  def on__embvar(token)
+  def on_embvar(token)
     token
   end
 
-  def on__float(token)
+  def on_float(token)
     token
   end
 
-  def on__gvar(token)
+  def on_gvar(token)
     token
   end
 
-  def on__heredoc_beg(token)
+  def on_heredoc_beg(token)
     token
   end
 
-  def on__heredoc_end(token)
+  def on_heredoc_end(token)
     token
   end
 
-  def on__ident(token)
+  def on_ident(token)
     token
   end
 
-  def on__ignored_nl(token)
+  def on_ignored_nl(token)
     token
   end
 
-  def on__int(token)
+  def on_int(token)
     token
   end
 
-  def on__ivar(token)
+  def on_ivar(token)
     token
   end
 
-  def on__kw(token)
+  def on_kw(token)
     token
   end
 
-  def on__lbrace(token)
+  def on_lbrace(token)
     token
   end
 
-  def on__lbracket(token)
+  def on_lbracket(token)
     token
   end
 
-  def on__lparen(token)
+  def on_lparen(token)
     token
   end
 
-  def on__nl(token)
+  def on_nl(token)
     token
   end
 
-  def on__op(token)
+  def on_op(token)
     token
   end
 
-  def on__period(token)
+  def on_period(token)
     token
   end
 
-  def on__qwords_beg(token)
+  def on_qwords_beg(token)
     token
   end
 
-  def on__rbrace(token)
+  def on_rbrace(token)
     token
   end
 
-  def on__rbracket(token)
+  def on_rbracket(token)
     token
   end
 
-  def on__regexp_beg(token)
+  def on_regexp_beg(token)
     token
   end
 
-  def on__regexp_end(token)
+  def on_regexp_end(token)
     token
   end
 
-  def on__rparen(token)
+  def on_rparen(token)
     token
   end
 
-  def on__semicolon(token)
+  def on_semicolon(token)
     token
   end
 
-  def on__sp(token)
+  def on_sp(token)
     token
   end
 
-  def on__symbeg(token)
+  def on_symbeg(token)
     token
   end
 
-  def on__tstring_beg(token)
+  def on_tstring_beg(token)
     token
   end
 
-  def on__tstring_content(token)
+  def on_tstring_content(token)
     token
   end
 
-  def on__tstring_end(token)
+  def on_tstring_end(token)
     token
   end
 
-  def on__words_beg(token)
+  def on_words_beg(token)
     token
   end
 
-  def on__words_sep(token)
+  def on_words_sep(token)
     token
   end
 end
+
+require 'ripper/tokenizer'

ext/ripper/lib/ripper/filter.rb

@@ -0,0 +1,55 @@
+#
+# ripper/filter.rb
+#
+# Copyright (C) 2004 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper'
+
+class Ripper
+
+  class Filter
+
+    def initialize(src, filename = '-', lineno = 1)
+      @__parser = Tokenizer.new(src, filename, lineno)
+      @__line = nil
+      @__col = nil
+    end
+
+    def filename
+      @__parser.filename
+    end
+
+    def lineno
+      @__line
+    end
+
+    def column
+      @__col
+    end
+
+    def parse(init)
+      data = init
+      @__parser.parse.each do |pos, event, tok|
+        @__line, @__col = *pos
+        data = if respond_to?(event, true)
+               then __send__(event, tok, data)
+               else on_default(event, tok, data)
+               end
+      end
+      data
+    end
+
+    private
+
+    def on_default(event, tok, data)
+      data
+    end
+
+  end
+
+end

ext/ripper/lib/ripper/tokenizer.rb

@@ -8,76 +8,55 @@
 # For details of Ruby License, see ruby/COPYING.
 #
 
-require 'ripper'
-
 class Ripper
 
-  def Ripper.tokenize(str)
+  # Tokenizes Ruby program and returns an Array of String.
-    Tokenizer.tokenize(str)
+  def Ripper.tokenize(src, filename = '-', lineno = 1)
+    Tokenizer.new(src, filename, lineno).tokenize
   end
 
-
+  # Tokenizes Ruby program and returns an Array of Array,
-  class Tokenizer < ::Ripper
+  # which is formatted like [[lineno, column], type, token].
-
+  #
-    def Tokenizer.tokenize(str, filename = '-', lineno = 1)
+  #   require 'ripper'
-      new(str, filename, lineno).tokenize
+  #   require 'pp'
-    end
+  #
-
+  #   p Ripper.scan("def m(a) nil end")
-    def initialize(src, filename = '-', lineno = 1)
+  #     #=> [[[1,  0], :on_kw,     "def"],
-      @src = src
+  #          [[1,  3], :on_sp,     " "  ],
-      @__filename = filename
+  #          [[1,  4], :on_ident,  "m"  ],
-      @__linestart = lineno
+  #          [[1,  5], :on_lparen, "("  ],
-      @__line = nil
+  #          [[1,  6], :on_ident,  "a"  ],
-      @__col = nil
+  #          [[1,  7], :on_rparen, ")"  ],
-    end
+  #          [[1,  8], :on_sp,     " "  ],
-
+  #          [[1,  9], :on_kw,     "nil"],
-    def filename
+  #          [[1, 12], :on_sp,     " "  ],
-      @__filename
+  #          [[1, 13], :on_kw,     "end"]]
-    end
+  #
-
+  def Ripper.scan(src, filename = '-', lineno = 1)
-    def lineno
+    Tokenizer.new(src, filename, lineno).parse
-      @__line
-    end
-
-    def column
-      @__col
   end
 
+  class Tokenizer < ::Ripper   #:nodoc: internal use only
     def tokenize
-      _exec_tokenizer().map {|pos, event, tok| tok }
+      parse().map {|pos, event, tok| tok }
     end
 
     def parse
-      _exec_tokenizer().each do |pos, event, tok|
+      @buf = []
-        @__line, @__col = *pos
-        on__scan(event, tok)
-        __send__(event, tok)
-      end
-    end
-
-    private
-
-    def _exec_tokenizer
-      TokenSorter.new(@src, @__filename, @__linestart).parse
-    end
-
-  end
-
-
-  class TokenSorter < ::Ripper   #:nodoc: internal use only
-
-    def parse
-      @data = []
       super
-      @data.sort_by {|pos, event, tok| pos }
+      @buf.sort_by {|pos, event, tok| pos }
     end
 
     private
 
-    def on__scan(event, tok)
+    SCANNER_EVENTS.each do |event|
-      @data.push [[lineno(),column()], event, tok]
+      module_eval(<<-End)
-    end
+        def on_#{event}(tok)
-
+          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        end
+      End
+    end
   end
 
 end

ext/ripper/ripper.rb.in

@@ -23,7 +23,6 @@ class Ripper
 
   # This table contains name of scanner events and its arity
   # (arity is always 1 for all scanner events).
-  # on__scan is NOT a scanner event.
   SCANNER_EVENT_TABLE = {
 #include ids2
   }
@@ -31,7 +30,7 @@ class Ripper
   # This array contains name of scanner events.
   SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
 
-  # This table contains name of all ripper events, except on__scan.
+  # This table contains name of all ripper events.
   EVENTS = PARSER_EVENTS + SCANNER_EVENTS
 
   ###                ###
@@ -57,8 +56,7 @@ class Ripper
   #
   # Lexer Events
   #
-
-  def on__scan(event, token)
-  end
 #include handlers2
 end
+
+require 'ripper/tokenizer'

ext/ripper/tools/generate-eventids1.rb

@@ -11,6 +11,6 @@ puts 'static void'
 puts 'ripper_init_eventids1()'
 puts '{'
 ids.each do |id|
-  puts %Q[    ripper_id_#{id} = rb_intern("on__#{id}");]
+  puts %Q[    ripper_id_#{id} = rb_intern("on_#{id}");]
 end
 puts '}'

ext/ripper/tools/generate-ripper_rb.rb

@@ -19,7 +19,7 @@ header
         id, arity = line.split
         arity = arity.to_i
         puts
-        puts "  def on__#{id}#{paramdecl(arity)}"
+        puts "  def on_#{id}#{paramdecl(arity)}"
         puts "    #{arity == 0 ? 'nil' : 'a'}"
         puts "  end"
       end
@@ -28,7 +28,7 @@ header
         id, arity = line.split
         arity = arity.to_i
         puts
-        puts "  def on__#{id}(token)"
+        puts "  def on_#{id}(token)"
         puts "    token"
         puts "  end"
       end

ext/ripper/tools/list-scan-event-ids.rb

@@ -17,7 +17,7 @@ def main
 end
 
 def extract_ids(f)
-  (f.read.scan(/ripper_id_(\w+)/).flatten - ['scan']).uniq
+  f.read.scan(/ripper_id_(\w+)/).flatten.uniq
 end
 
 main

parse.y

@@ -4203,14 +4203,11 @@ ripper_dispatch_scan_event(parser, t)
     int t;
 {
     VALUE str;
-    ID event;
 
     if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp");
     if (lex_p == parser->tokp) return;
     str = rb_str_new(parser->tokp, lex_p - parser->tokp);
-    event = ripper_token2eventid(t);
+    yylval.val = ripper_dispatch1(parser, ripper_token2eventid(t), str);
-    ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
-    yylval.val = ripper_dispatch1(parser, event, str);
     ripper_flush(parser);
 }
 
@@ -4221,13 +4218,10 @@ ripper_dispatch_delayed_token(parser, t)
 {
     int saved_line = ruby_sourceline;
     char *saved_tokp = parser->tokp;
-    ID event = ripper_token2eventid(t);
-    VALUE str = parser->delayed;
 
     ruby_sourceline = parser->delayed_line;
     parser->tokp = lex_pbeg + parser->delayed_col;
-    ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
+    yylval.val = ripper_dispatch1(parser, ripper_token2eventid(t), parser->delayed);
-    yylval.val = ripper_dispatch1(parser, event, str);
     parser->delayed = Qnil;
     ruby_sourceline = saved_line;
     parser->tokp = saved_tokp;

sample/ripper/colorize.rb

@@ -0,0 +1,30 @@
+require 'ripper/filter'
+
+class ColorizeFilter < Ripper::Filter
+  def on_default(event, tok, f)
+    f << escape(tok)
+  end
+
+  def on_comment(tok, f)
+    f << %Q[<span class="comment">#{escape(tok)}</span>]
+  end
+
+  def on_tstring_content(tok, f)
+    f << %Q[<span class="string">#{escape(tok)}</span>]
+  end
+
+  ESC = {
+    '&' => '&amp;',
+    '<' => '&lt;',
+    '>' => '&gt;'
+  }
+
+  def escape(str)
+    tbl = ESC
+    str.gsub(/[&<>]/) {|ch| tbl[ch] }
+  end
+end
+
+if $0 == __FILE__
+  ColorizeFilter.new(ARGF).parse($stdout)
+end

sample/ripper/strip-comment.rb

@@ -0,0 +1,19 @@
+# $Id$
+
+require 'ripper/filter'
+
+class CommentStripper < Ripper::Filter
+  def CommentStripper.strip(src)
+    new(src).parse(nil)
+  end
+
+  def on_default(event, token, data)
+    print token
+  end
+
+  def on_comment(token, data)
+    puts
+  end
+end
+
+CommentStripper.strip(ARGF)