diff --git a/ChangeLog b/ChangeLog index 6713e94dfa..f10d6e28b5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,19 @@ +Thu Sep 23 19:48:14 2004 Minero Aoki + + * ext/ripper/Makefile.dev: removed. + + * ext/ripper/ripper.rb.in: moved to lib/ripper/core.rb.in. + + * ext/ripper/lib/ripper/core.rb: new file. + + * ext/ripper/lib/ripper/core.rb.in: new file. + + * ext/ripper/tools/generate-ripper_rb.rb: change comment. + + * test/ripper/*.rb: on__scan event removed. + + * test/ripper/*.rb: event name is changed: on__XXX -> on_XXX. + Thu Sep 23 09:29:14 2004 Yukihiro Matsumoto * hash.c (rb_hash_rehash): replace st_foreach() by its deep diff --git a/ext/ripper/MANIFEST b/ext/ripper/MANIFEST index 25a907b3df..9ca4fa82ca 100644 --- a/ext/ripper/MANIFEST +++ b/ext/ripper/MANIFEST @@ -1,11 +1,11 @@ MANIFEST -Makefile.dev README depend eventids2.c extconf.rb -ripper.rb.in lib/ripper.rb +lib/ripper/core.rb +lib/ripper/core.rb.in lib/ripper/filter.rb lib/ripper/tokenizer.rb test/check-event-arity.rb diff --git a/ext/ripper/Makefile.dev b/ext/ripper/Makefile.dev deleted file mode 100644 index ab16a7a74b..0000000000 --- a/ext/ripper/Makefile.dev +++ /dev/null @@ -1,18 +0,0 @@ -# Makefile for ripper developpers -# !!! This file requires GNU make !!! - -include Makefile - -lib/ripper.rb: $(srcdir)/ripper.rb.in ids1 ids2 $(srcdir)/tools/generate-ripper_rb.rb - $(RUBY) $(srcdir)/tools/generate-ripper_rb.rb $(srcdir)/ripper.rb.in ids1 ids2 > $@ - -test: - $(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y - sh test/check-event-coverage.sh - $(RUBY) test/check-scanner-event-coverage.rb - $(RUBY) test/validate.rb > /dev/null - @echo OK - -preproc: ripper.E -ripper.E: ripper.c - $(CC) -E $(CPPFLAGS) ripper.c | ruby $(srcdir)/tools/strip.rb > $@ diff --git a/ext/ripper/depend b/ext/ripper/depend index b0ab953a94..3cfc4ab404 100644 --- a/ext/ripper/depend +++ b/ext/ripper/depend @@ -14,3 +14,22 @@ ids1: $(srcdir)/tools/list-parse-event-ids.rb $(hdrdir)/parse.y ids2: $(srcdir)/tools/list-scan-event-ids.rb $(srcdir)/eventids2.c $(RUBY) $(srcdir)/tools/list-scan-event-ids.rb -a $(srcdir)/eventids2.c > $@ + +rb: $(srcdir)/lib/ripper/core.rb +$(srcdir)/lib/ripper/core.rb: $(srcdir)/lib/ripper/core.rb.in ids1 ids2 $(srcdir)/tools/generate-ripper_rb.rb + $(RUBY) $(srcdir)/tools/generate-ripper_rb.rb $(srcdir)/lib/ripper/core.rb.in ids1 ids2 > $@ + +# +# Entries for only ripper developpers: DO NOT USE +# + +test: + $(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y + sh test/check-event-coverage.sh + $(RUBY) test/check-scanner-event-coverage.rb + $(RUBY) test/validate.rb > /dev/null + @echo OK + +preproc: ripper.E +ripper.E: ripper.c + $(CC) -E $(CPPFLAGS) ripper.c | $(RUBY) $(srcdir)/tools/strip.rb > $@ diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb index cb4924d0cd..38a5a2bd19 100644 --- a/ext/ripper/lib/ripper.rb +++ b/ext/ripper/lib/ripper.rb @@ -1,877 +1,3 @@ -# -# This file is automatically generated from ripper.rb.in and parse.y. -# DO NOT MODIFY!!!!!! -# -# -# ripper.rb -# -# Copyright (C) 2003,2004 Minero Aoki -# - -require 'ripper.so' - -class Ripper - # Parses Ruby program read from _src_. - # _src_ must be a String or a IO or a object which has #gets method. - def Ripper.parse(src, filename = '(ripper)', lineno = 1) - new(src, filename, lineno).parse - end - - # This table contains name of parser events and its arity. - PARSER_EVENT_TABLE = { - :BEGIN => 1, - :END => 1, - :alias => 2, - :alias_error => 1, - :aref => 2, - :aref_field => 2, - :arg_ambiguous => 0, - :arg_paren => 1, - :arglist_add => 2, - :arglist_add_block => 2, - :arglist_add_star => 2, - :arglist_new => 0, - :arglist_prepend => 2, - :array => 1, - :assign => 2, - :assign_error => 1, - :assoc_new => 2, - :assoclist_from_args => 1, - :bare_assoc_hash => 1, - :begin => 1, - :binary => 3, - :blockvar_add_block => 2, - :blockvar_add_star => 2, - :blockvar_new => 1, - :bodystmt => 4, - :brace_block => 2, - :break => 1, - :call => 3, - :case => 2, - :class => 3, - :class_name_error => 1, - :command => 2, - :command_call => 4, - :const_ref => 1, - :constpath_field => 2, - :constpath_ref => 2, - :def => 3, - :defined => 1, - :defs => 5, - :do_block => 2, - :dot2 => 2, - :dot3 => 2, - :dyna_symbol => 1, - :else => 1, - :elsif => 3, - :ensure => 1, - :fcall => 1, - :field => 3, - :for => 3, - :hash => 1, - :if => 3, - :if_mod => 2, - :ifop => 3, - :iter_block => 2, - :massign => 2, - :method_add_arg => 2, - :mlhs_add => 2, - :mlhs_add_star => 2, - :mlhs_new => 0, - :mlhs_paren => 1, - :module => 2, - :mrhs_add => 2, - :mrhs_add_star => 2, - :mrhs_new => 0, - :mrhs_new_from_arglist => 1, - :next => 1, - :opassign => 3, - :param_error => 1, - :params => 4, - :paren => 1, - :parse_error => 1, - :program => 1, - :qwords_add => 2, - :qwords_new => 0, - :redo => 0, - :regexp_literal => 1, - :rescue => 4, - :rescue_mod => 2, - :restparam => 1, - :retry => 0, - :return => 1, - :return0 => 0, - :sclass => 2, - :space => 1, - :stmts_add => 2, - :stmts_new => 0, - :string_add => 2, - :string_concat => 2, - :string_content => 0, - :string_dvar => 1, - :string_embexpr => 1, - :string_literal => 1, - :super => 1, - :symbol => 1, - :symbol_literal => 1, - :topconst_field => 1, - :topconst_ref => 1, - :unary => 2, - :undef => 1, - :unless => 3, - :unless_mod => 2, - :until => 2, - :until_mod => 2, - :var_alias => 2, - :var_field => 1, - :var_ref => 1, - :void_stmt => 0, - :when => 3, - :while => 2, - :while_mod => 2, - :word_add => 2, - :word_new => 0, - :words_add => 2, - :words_new => 0, - :xstring_add => 2, - :xstring_literal => 1, - :xstring_new => 0, - :yield => 1, - :yield0 => 0, - :zsuper => 0 - } - - # This array contains name of parser events. - PARSER_EVENTS = PARSER_EVENT_TABLE.keys - - # This table contains name of scanner events and its arity - # (arity is always 1 for all scanner events). - SCANNER_EVENT_TABLE = { - :CHAR => 1, - :__end__ => 1, - :backref => 1, - :backtick => 1, - :comma => 1, - :comment => 1, - :const => 1, - :cvar => 1, - :embdoc => 1, - :embdoc_beg => 1, - :embdoc_end => 1, - :embexpr_beg => 1, - :embexpr_end => 1, - :embvar => 1, - :float => 1, - :gvar => 1, - :heredoc_beg => 1, - :heredoc_end => 1, - :ident => 1, - :ignored_nl => 1, - :int => 1, - :ivar => 1, - :kw => 1, - :lbrace => 1, - :lbracket => 1, - :lparen => 1, - :nl => 1, - :op => 1, - :period => 1, - :qwords_beg => 1, - :rbrace => 1, - :rbracket => 1, - :regexp_beg => 1, - :regexp_end => 1, - :rparen => 1, - :semicolon => 1, - :sp => 1, - :symbeg => 1, - :tstring_beg => 1, - :tstring_content => 1, - :tstring_end => 1, - :words_beg => 1, - :words_sep => 1 - } - - # This array contains name of scanner events. - SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys - - # This table contains name of all ripper events. - EVENTS = PARSER_EVENTS + SCANNER_EVENTS - - ### ### - ### Event Handlers ### - ### ### - - private - - def warn(fmt, *args) - end - - def warning(fmt, *args) - end - - def compile_error(msg) - end - - # - # Parser Events - # - - def on_BEGIN(a) - a - end - - def on_END(a) - a - end - - def on_alias(a, b) - a - end - - def on_alias_error(a) - a - end - - def on_aref(a, b) - a - end - - def on_aref_field(a, b) - a - end - - def on_arg_ambiguous - nil - end - - def on_arg_paren(a) - a - end - - def on_arglist_add(a, b) - a - end - - def on_arglist_add_block(a, b) - a - end - - def on_arglist_add_star(a, b) - a - end - - def on_arglist_new - nil - end - - def on_arglist_prepend(a, b) - a - end - - def on_array(a) - a - end - - def on_assign(a, b) - a - end - - def on_assign_error(a) - a - end - - def on_assoc_new(a, b) - a - end - - def on_assoclist_from_args(a) - a - end - - def on_bare_assoc_hash(a) - a - end - - def on_begin(a) - a - end - - def on_binary(a, b, c) - a - end - - def on_blockvar_add_block(a, b) - a - end - - def on_blockvar_add_star(a, b) - a - end - - def on_blockvar_new(a) - a - end - - def on_bodystmt(a, b, c, d) - a - end - - def on_brace_block(a, b) - a - end - - def on_break(a) - a - end - - def on_call(a, b, c) - a - end - - def on_case(a, b) - a - end - - def on_class(a, b, c) - a - end - - def on_class_name_error(a) - a - end - - def on_command(a, b) - a - end - - def on_command_call(a, b, c, d) - a - end - - def on_const_ref(a) - a - end - - def on_constpath_field(a, b) - a - end - - def on_constpath_ref(a, b) - a - end - - def on_def(a, b, c) - a - end - - def on_defined(a) - a - end - - def on_defs(a, b, c, d, e) - a - end - - def on_do_block(a, b) - a - end - - def on_dot2(a, b) - a - end - - def on_dot3(a, b) - a - end - - def on_dyna_symbol(a) - a - end - - def on_else(a) - a - end - - def on_elsif(a, b, c) - a - end - - def on_ensure(a) - a - end - - def on_fcall(a) - a - end - - def on_field(a, b, c) - a - end - - def on_for(a, b, c) - a - end - - def on_hash(a) - a - end - - def on_if(a, b, c) - a - end - - def on_if_mod(a, b) - a - end - - def on_ifop(a, b, c) - a - end - - def on_iter_block(a, b) - a - end - - def on_massign(a, b) - a - end - - def on_method_add_arg(a, b) - a - end - - def on_mlhs_add(a, b) - a - end - - def on_mlhs_add_star(a, b) - a - end - - def on_mlhs_new - nil - end - - def on_mlhs_paren(a) - a - end - - def on_module(a, b) - a - end - - def on_mrhs_add(a, b) - a - end - - def on_mrhs_add_star(a, b) - a - end - - def on_mrhs_new - nil - end - - def on_mrhs_new_from_arglist(a) - a - end - - def on_next(a) - a - end - - def on_opassign(a, b, c) - a - end - - def on_param_error(a) - a - end - - def on_params(a, b, c, d) - a - end - - def on_paren(a) - a - end - - def on_parse_error(a) - a - end - - def on_program(a) - a - end - - def on_qwords_add(a, b) - a - end - - def on_qwords_new - nil - end - - def on_redo - nil - end - - def on_regexp_literal(a) - a - end - - def on_rescue(a, b, c, d) - a - end - - def on_rescue_mod(a, b) - a - end - - def on_restparam(a) - a - end - - def on_retry - nil - end - - def on_return(a) - a - end - - def on_return0 - nil - end - - def on_sclass(a, b) - a - end - - def on_space(a) - a - end - - def on_stmts_add(a, b) - a - end - - def on_stmts_new - nil - end - - def on_string_add(a, b) - a - end - - def on_string_concat(a, b) - a - end - - def on_string_content - nil - end - - def on_string_dvar(a) - a - end - - def on_string_embexpr(a) - a - end - - def on_string_literal(a) - a - end - - def on_super(a) - a - end - - def on_symbol(a) - a - end - - def on_symbol_literal(a) - a - end - - def on_topconst_field(a) - a - end - - def on_topconst_ref(a) - a - end - - def on_unary(a, b) - a - end - - def on_undef(a) - a - end - - def on_unless(a, b, c) - a - end - - def on_unless_mod(a, b) - a - end - - def on_until(a, b) - a - end - - def on_until_mod(a, b) - a - end - - def on_var_alias(a, b) - a - end - - def on_var_field(a) - a - end - - def on_var_ref(a) - a - end - - def on_void_stmt - nil - end - - def on_when(a, b, c) - a - end - - def on_while(a, b) - a - end - - def on_while_mod(a, b) - a - end - - def on_word_add(a, b) - a - end - - def on_word_new - nil - end - - def on_words_add(a, b) - a - end - - def on_words_new - nil - end - - def on_xstring_add(a, b) - a - end - - def on_xstring_literal(a) - a - end - - def on_xstring_new - nil - end - - def on_yield(a) - a - end - - def on_yield0 - nil - end - - def on_zsuper - nil - end - - # - # Lexer Events - # - - def on_CHAR(token) - token - end - - def on___end__(token) - token - end - - def on_backref(token) - token - end - - def on_backtick(token) - token - end - - def on_comma(token) - token - end - - def on_comment(token) - token - end - - def on_const(token) - token - end - - def on_cvar(token) - token - end - - def on_embdoc(token) - token - end - - def on_embdoc_beg(token) - token - end - - def on_embdoc_end(token) - token - end - - def on_embexpr_beg(token) - token - end - - def on_embexpr_end(token) - token - end - - def on_embvar(token) - token - end - - def on_float(token) - token - end - - def on_gvar(token) - token - end - - def on_heredoc_beg(token) - token - end - - def on_heredoc_end(token) - token - end - - def on_ident(token) - token - end - - def on_ignored_nl(token) - token - end - - def on_int(token) - token - end - - def on_ivar(token) - token - end - - def on_kw(token) - token - end - - def on_lbrace(token) - token - end - - def on_lbracket(token) - token - end - - def on_lparen(token) - token - end - - def on_nl(token) - token - end - - def on_op(token) - token - end - - def on_period(token) - token - end - - def on_qwords_beg(token) - token - end - - def on_rbrace(token) - token - end - - def on_rbracket(token) - token - end - - def on_regexp_beg(token) - token - end - - def on_regexp_end(token) - token - end - - def on_rparen(token) - token - end - - def on_semicolon(token) - token - end - - def on_sp(token) - token - end - - def on_symbeg(token) - token - end - - def on_tstring_beg(token) - token - end - - def on_tstring_content(token) - token - end - - def on_tstring_end(token) - token - end - - def on_words_beg(token) - token - end - - def on_words_sep(token) - token - end -end - +require 'ripper/core' require 'ripper/tokenizer' +require 'ripper/filter' diff --git a/ext/ripper/lib/ripper/core.rb b/ext/ripper/lib/ripper/core.rb new file mode 100644 index 0000000000..e936783698 --- /dev/null +++ b/ext/ripper/lib/ripper/core.rb @@ -0,0 +1,883 @@ +# This file is automatically generated from core.rb.in and parse.y. +# DO NOT MODIFY!!!!!! + +# +# ripper/core.rb +# +# Copyright (C) 2003,2004 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper.so' + +class Ripper + # Parses Ruby program read from _src_. + # _src_ must be a String or a IO or a object which has #gets method. + def Ripper.parse(src, filename = '(ripper)', lineno = 1) + new(src, filename, lineno).parse + end + + # This table contains name of parser events and its arity. + PARSER_EVENT_TABLE = { + :BEGIN => 1, + :END => 1, + :alias => 2, + :alias_error => 1, + :aref => 2, + :aref_field => 2, + :arg_ambiguous => 0, + :arg_paren => 1, + :arglist_add => 2, + :arglist_add_block => 2, + :arglist_add_star => 2, + :arglist_new => 0, + :arglist_prepend => 2, + :array => 1, + :assign => 2, + :assign_error => 1, + :assoc_new => 2, + :assoclist_from_args => 1, + :bare_assoc_hash => 1, + :begin => 1, + :binary => 3, + :blockvar_add_block => 2, + :blockvar_add_star => 2, + :blockvar_new => 1, + :bodystmt => 4, + :brace_block => 2, + :break => 1, + :call => 3, + :case => 2, + :class => 3, + :class_name_error => 1, + :command => 2, + :command_call => 4, + :const_ref => 1, + :constpath_field => 2, + :constpath_ref => 2, + :def => 3, + :defined => 1, + :defs => 5, + :do_block => 2, + :dot2 => 2, + :dot3 => 2, + :dyna_symbol => 1, + :else => 1, + :elsif => 3, + :ensure => 1, + :fcall => 1, + :field => 3, + :for => 3, + :hash => 1, + :if => 3, + :if_mod => 2, + :ifop => 3, + :iter_block => 2, + :massign => 2, + :method_add_arg => 2, + :mlhs_add => 2, + :mlhs_add_star => 2, + :mlhs_new => 0, + :mlhs_paren => 1, + :module => 2, + :mrhs_add => 2, + :mrhs_add_star => 2, + :mrhs_new => 0, + :mrhs_new_from_arglist => 1, + :next => 1, + :opassign => 3, + :param_error => 1, + :params => 4, + :paren => 1, + :parse_error => 1, + :program => 1, + :qwords_add => 2, + :qwords_new => 0, + :redo => 0, + :regexp_literal => 1, + :rescue => 4, + :rescue_mod => 2, + :restparam => 1, + :retry => 0, + :return => 1, + :return0 => 0, + :sclass => 2, + :space => 1, + :stmts_add => 2, + :stmts_new => 0, + :string_add => 2, + :string_concat => 2, + :string_content => 0, + :string_dvar => 1, + :string_embexpr => 1, + :string_literal => 1, + :super => 1, + :symbol => 1, + :symbol_literal => 1, + :topconst_field => 1, + :topconst_ref => 1, + :unary => 2, + :undef => 1, + :unless => 3, + :unless_mod => 2, + :until => 2, + :until_mod => 2, + :var_alias => 2, + :var_field => 1, + :var_ref => 1, + :void_stmt => 0, + :when => 3, + :while => 2, + :while_mod => 2, + :word_add => 2, + :word_new => 0, + :words_add => 2, + :words_new => 0, + :xstring_add => 2, + :xstring_literal => 1, + :xstring_new => 0, + :yield => 1, + :yield0 => 0, + :zsuper => 0 + } + + # This array contains name of parser events. + PARSER_EVENTS = PARSER_EVENT_TABLE.keys + + # This table contains name of scanner events and its arity + # (arity is always 1 for all scanner events). + SCANNER_EVENT_TABLE = { + :CHAR => 1, + :__end__ => 1, + :backref => 1, + :backtick => 1, + :comma => 1, + :comment => 1, + :const => 1, + :cvar => 1, + :embdoc => 1, + :embdoc_beg => 1, + :embdoc_end => 1, + :embexpr_beg => 1, + :embexpr_end => 1, + :embvar => 1, + :float => 1, + :gvar => 1, + :heredoc_beg => 1, + :heredoc_end => 1, + :ident => 1, + :ignored_nl => 1, + :int => 1, + :ivar => 1, + :kw => 1, + :lbrace => 1, + :lbracket => 1, + :lparen => 1, + :nl => 1, + :op => 1, + :period => 1, + :qwords_beg => 1, + :rbrace => 1, + :rbracket => 1, + :regexp_beg => 1, + :regexp_end => 1, + :rparen => 1, + :semicolon => 1, + :sp => 1, + :symbeg => 1, + :tstring_beg => 1, + :tstring_content => 1, + :tstring_end => 1, + :words_beg => 1, + :words_sep => 1 + } + + # This array contains name of scanner events. + SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys + + # This table contains name of all ripper events. + EVENTS = PARSER_EVENTS + SCANNER_EVENTS + + ### ### + ### Event Handlers ### + ### ### + + private + + # This method is called when weak warning is produced by the parser. + # _fmt_ and _args_ is printf style. + def warn(fmt, *args) + end + + # This method is called when strong warning is produced by the parser. + # _fmt_ and _args_ is printf style. + def warning(fmt, *args) + end + + # This method is called when the parser found syntax error. + def compile_error(msg) + end + + # + # Parser Events + # + + def on_BEGIN(a) + a + end + + def on_END(a) + a + end + + def on_alias(a, b) + a + end + + def on_alias_error(a) + a + end + + def on_aref(a, b) + a + end + + def on_aref_field(a, b) + a + end + + def on_arg_ambiguous + nil + end + + def on_arg_paren(a) + a + end + + def on_arglist_add(a, b) + a + end + + def on_arglist_add_block(a, b) + a + end + + def on_arglist_add_star(a, b) + a + end + + def on_arglist_new + nil + end + + def on_arglist_prepend(a, b) + a + end + + def on_array(a) + a + end + + def on_assign(a, b) + a + end + + def on_assign_error(a) + a + end + + def on_assoc_new(a, b) + a + end + + def on_assoclist_from_args(a) + a + end + + def on_bare_assoc_hash(a) + a + end + + def on_begin(a) + a + end + + def on_binary(a, b, c) + a + end + + def on_blockvar_add_block(a, b) + a + end + + def on_blockvar_add_star(a, b) + a + end + + def on_blockvar_new(a) + a + end + + def on_bodystmt(a, b, c, d) + a + end + + def on_brace_block(a, b) + a + end + + def on_break(a) + a + end + + def on_call(a, b, c) + a + end + + def on_case(a, b) + a + end + + def on_class(a, b, c) + a + end + + def on_class_name_error(a) + a + end + + def on_command(a, b) + a + end + + def on_command_call(a, b, c, d) + a + end + + def on_const_ref(a) + a + end + + def on_constpath_field(a, b) + a + end + + def on_constpath_ref(a, b) + a + end + + def on_def(a, b, c) + a + end + + def on_defined(a) + a + end + + def on_defs(a, b, c, d, e) + a + end + + def on_do_block(a, b) + a + end + + def on_dot2(a, b) + a + end + + def on_dot3(a, b) + a + end + + def on_dyna_symbol(a) + a + end + + def on_else(a) + a + end + + def on_elsif(a, b, c) + a + end + + def on_ensure(a) + a + end + + def on_fcall(a) + a + end + + def on_field(a, b, c) + a + end + + def on_for(a, b, c) + a + end + + def on_hash(a) + a + end + + def on_if(a, b, c) + a + end + + def on_if_mod(a, b) + a + end + + def on_ifop(a, b, c) + a + end + + def on_iter_block(a, b) + a + end + + def on_massign(a, b) + a + end + + def on_method_add_arg(a, b) + a + end + + def on_mlhs_add(a, b) + a + end + + def on_mlhs_add_star(a, b) + a + end + + def on_mlhs_new + nil + end + + def on_mlhs_paren(a) + a + end + + def on_module(a, b) + a + end + + def on_mrhs_add(a, b) + a + end + + def on_mrhs_add_star(a, b) + a + end + + def on_mrhs_new + nil + end + + def on_mrhs_new_from_arglist(a) + a + end + + def on_next(a) + a + end + + def on_opassign(a, b, c) + a + end + + def on_param_error(a) + a + end + + def on_params(a, b, c, d) + a + end + + def on_paren(a) + a + end + + def on_parse_error(a) + a + end + + def on_program(a) + a + end + + def on_qwords_add(a, b) + a + end + + def on_qwords_new + nil + end + + def on_redo + nil + end + + def on_regexp_literal(a) + a + end + + def on_rescue(a, b, c, d) + a + end + + def on_rescue_mod(a, b) + a + end + + def on_restparam(a) + a + end + + def on_retry + nil + end + + def on_return(a) + a + end + + def on_return0 + nil + end + + def on_sclass(a, b) + a + end + + def on_space(a) + a + end + + def on_stmts_add(a, b) + a + end + + def on_stmts_new + nil + end + + def on_string_add(a, b) + a + end + + def on_string_concat(a, b) + a + end + + def on_string_content + nil + end + + def on_string_dvar(a) + a + end + + def on_string_embexpr(a) + a + end + + def on_string_literal(a) + a + end + + def on_super(a) + a + end + + def on_symbol(a) + a + end + + def on_symbol_literal(a) + a + end + + def on_topconst_field(a) + a + end + + def on_topconst_ref(a) + a + end + + def on_unary(a, b) + a + end + + def on_undef(a) + a + end + + def on_unless(a, b, c) + a + end + + def on_unless_mod(a, b) + a + end + + def on_until(a, b) + a + end + + def on_until_mod(a, b) + a + end + + def on_var_alias(a, b) + a + end + + def on_var_field(a) + a + end + + def on_var_ref(a) + a + end + + def on_void_stmt + nil + end + + def on_when(a, b, c) + a + end + + def on_while(a, b) + a + end + + def on_while_mod(a, b) + a + end + + def on_word_add(a, b) + a + end + + def on_word_new + nil + end + + def on_words_add(a, b) + a + end + + def on_words_new + nil + end + + def on_xstring_add(a, b) + a + end + + def on_xstring_literal(a) + a + end + + def on_xstring_new + nil + end + + def on_yield(a) + a + end + + def on_yield0 + nil + end + + def on_zsuper + nil + end + + # + # Lexer Events + # + + def on_CHAR(token) + token + end + + def on___end__(token) + token + end + + def on_backref(token) + token + end + + def on_backtick(token) + token + end + + def on_comma(token) + token + end + + def on_comment(token) + token + end + + def on_const(token) + token + end + + def on_cvar(token) + token + end + + def on_embdoc(token) + token + end + + def on_embdoc_beg(token) + token + end + + def on_embdoc_end(token) + token + end + + def on_embexpr_beg(token) + token + end + + def on_embexpr_end(token) + token + end + + def on_embvar(token) + token + end + + def on_float(token) + token + end + + def on_gvar(token) + token + end + + def on_heredoc_beg(token) + token + end + + def on_heredoc_end(token) + token + end + + def on_ident(token) + token + end + + def on_ignored_nl(token) + token + end + + def on_int(token) + token + end + + def on_ivar(token) + token + end + + def on_kw(token) + token + end + + def on_lbrace(token) + token + end + + def on_lbracket(token) + token + end + + def on_lparen(token) + token + end + + def on_nl(token) + token + end + + def on_op(token) + token + end + + def on_period(token) + token + end + + def on_qwords_beg(token) + token + end + + def on_rbrace(token) + token + end + + def on_rbracket(token) + token + end + + def on_regexp_beg(token) + token + end + + def on_regexp_end(token) + token + end + + def on_rparen(token) + token + end + + def on_semicolon(token) + token + end + + def on_sp(token) + token + end + + def on_symbeg(token) + token + end + + def on_tstring_beg(token) + token + end + + def on_tstring_content(token) + token + end + + def on_tstring_end(token) + token + end + + def on_words_beg(token) + token + end + + def on_words_sep(token) + token + end +end diff --git a/ext/ripper/ripper.rb.in b/ext/ripper/lib/ripper/core.rb.in similarity index 70% rename from ext/ripper/ripper.rb.in rename to ext/ripper/lib/ripper/core.rb.in index 295dd1e353..7e6d51ab8f 100644 --- a/ext/ripper/ripper.rb.in +++ b/ext/ripper/lib/ripper/core.rb.in @@ -1,8 +1,12 @@ # -# ripper.rb +# ripper/core.rb # # Copyright (C) 2003,2004 Minero Aoki # +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# require 'ripper.so' @@ -39,12 +43,17 @@ class Ripper private + # This method is called when weak warning is produced by the parser. + # _fmt_ and _args_ is printf style. def warn(fmt, *args) end + # This method is called when strong warning is produced by the parser. + # _fmt_ and _args_ is printf style. def warning(fmt, *args) end + # This method is called when the parser found syntax error. def compile_error(msg) end @@ -58,5 +67,3 @@ class Ripper # #include handlers2 end - -require 'ripper/tokenizer' diff --git a/ext/ripper/lib/ripper/filter.rb b/ext/ripper/lib/ripper/filter.rb index 399c4c64bc..d1c7c4a45e 100644 --- a/ext/ripper/lib/ripper/filter.rb +++ b/ext/ripper/lib/ripper/filter.rb @@ -8,10 +8,12 @@ # For details of Ruby License, see ruby/COPYING. # -require 'ripper' +require 'ripper/core' class Ripper + # This class handles only scanner events, + # and they are dispatched in the `right' order (same with input). class Filter def initialize(src, filename = '-', lineno = 1) @@ -20,18 +22,27 @@ class Ripper @__col = nil end + # The file name of the input. def filename @__parser.filename end + # The line number of the current token. + # This value starts from 1. + # This method is valid only in event handlers. def lineno @__line end + # The column number of the current token. + # This value starts from 0. + # This method is valid only in event handlers. def column @__col end + # Starts parsing. _init_ is a data accumulator. + # It is passed to the next event handler (as of Enumerable#inject). def parse(init) data = init @__parser.parse.each do |pos, event, tok| @@ -46,7 +57,11 @@ class Ripper private - def on_default(event, tok, data) + # This method is called when some event handler have not defined. + # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data + # accumulator. The return value of this method is passed to the + # next event handler (as of Enumerable#inject). + def on_default(event, token, data) data end diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb index 4d90f16aa1..4209903ba7 100644 --- a/ext/ripper/lib/ripper/tokenizer.rb +++ b/ext/ripper/lib/ripper/tokenizer.rb @@ -8,6 +8,8 @@ # For details of Ruby License, see ruby/COPYING. # +require 'ripper/core' + class Ripper # Tokenizes Ruby program and returns an Array of String. diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb index 8a96e4cb1e..5c06ca40a4 100755 --- a/ext/ripper/tools/generate-ripper_rb.rb +++ b/ext/ripper/tools/generate-ripper_rb.rb @@ -3,10 +3,9 @@ def main template, ids1, ids2 = *ARGV print <
$"), - R.scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")') + scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")') end def test_ident assert_equal [], - R.scan('ident', '') + scan('ident', '') assert_equal ['lvar'], - R.scan('ident', 'lvar') + scan('ident', 'lvar') assert_equal ['m', 'lvar'], - R.scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)') + scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)') end def test_int assert_equal [], - R.scan('int', '') + scan('int', '') assert_equal ['1', '10', '100000000000000'], - R.scan('int', 'm(1,10,100000000000000)') + scan('int', 'm(1,10,100000000000000)') end def test_ivar assert_equal [], - R.scan('ivar', '') + scan('ivar', '') assert_equal ['@ivar'], - R.scan('ivar', '@ivar') + scan('ivar', '@ivar') assert_equal ['@__ivar__'], - R.scan('ivar', '@__ivar__') + scan('ivar', '@__ivar__') assert_equal ['@IVAR'], - R.scan('ivar', '@IVAR') + scan('ivar', '@IVAR') assert_equal ['@ivar'], - R.scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)') + scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)') end def test_kw assert_equal [], - R.scan('kw', '') + scan('kw', '') assert_equal %w(not), - R.scan('kw', 'not 1') + scan('kw', 'not 1') assert_equal %w(and), - R.scan('kw', '1 and 2') + scan('kw', '1 and 2') assert_equal %w(or), - R.scan('kw', '1 or 2') + scan('kw', '1 or 2') assert_equal %w(if then else end), - R.scan('kw', 'if 1 then 2 else 3 end') + scan('kw', 'if 1 then 2 else 3 end') assert_equal %w(if then elsif else end), - R.scan('kw', 'if 1 then 2 elsif 3 else 4 end') + scan('kw', 'if 1 then 2 elsif 3 else 4 end') assert_equal %w(unless then end), - R.scan('kw', 'unless 1 then end') + scan('kw', 'unless 1 then end') assert_equal %w(if true), - R.scan('kw', '1 if true') + scan('kw', '1 if true') assert_equal %w(unless false), - R.scan('kw', '2 unless false') + scan('kw', '2 unless false') assert_equal %w(case when when else end), - R.scan('kw', 'case n; when 1; when 2; else 3 end') + scan('kw', 'case n; when 1; when 2; else 3 end') assert_equal %w(while do nil end), - R.scan('kw', 'while 1 do nil end') + scan('kw', 'while 1 do nil end') assert_equal %w(until do nil end), - R.scan('kw', 'until 1 do nil end') + scan('kw', 'until 1 do nil end') assert_equal %w(while), - R.scan('kw', '1 while 2') + scan('kw', '1 while 2') assert_equal %w(until), - R.scan('kw', '1 until 2') + scan('kw', '1 until 2') assert_equal %w(while break next retry end), - R.scan('kw', 'while 1; break; next; retry end') + scan('kw', 'while 1; break; next; retry end') assert_equal %w(for in next break end), - R.scan('kw', 'for x in obj; next 1; break 2 end') + scan('kw', 'for x in obj; next 1; break 2 end') assert_equal %w(begin rescue retry end), - R.scan('kw', 'begin 1; rescue; retry; end') + scan('kw', 'begin 1; rescue; retry; end') assert_equal %w(rescue), - R.scan('kw', '1 rescue 2') + scan('kw', '1 rescue 2') assert_equal %w(def redo return end), - R.scan('kw', 'def m() redo; return end') + scan('kw', 'def m() redo; return end') assert_equal %w(def yield yield end), - R.scan('kw', 'def m() yield; yield 1 end') + scan('kw', 'def m() yield; yield 1 end') assert_equal %w(def super super super end), - R.scan('kw', 'def m() super; super(); super(1) end') + scan('kw', 'def m() super; super(); super(1) end') assert_equal %w(alias), - R.scan('kw', 'alias a b') + scan('kw', 'alias a b') assert_equal %w(undef), - R.scan('kw', 'undef public') + scan('kw', 'undef public') assert_equal %w(class end), - R.scan('kw', 'class A < Object; end') + scan('kw', 'class A < Object; end') assert_equal %w(module end), - R.scan('kw', 'module M; end') + scan('kw', 'module M; end') assert_equal %w(class end), - R.scan('kw', 'class << obj; end') + scan('kw', 'class << obj; end') assert_equal %w(BEGIN), - R.scan('kw', 'BEGIN { }') + scan('kw', 'BEGIN { }') assert_equal %w(END), - R.scan('kw', 'END { }') + scan('kw', 'END { }') assert_equal %w(self), - R.scan('kw', 'self.class') + scan('kw', 'self.class') assert_equal %w(nil true false), - R.scan('kw', 'p(nil, true, false)') + scan('kw', 'p(nil, true, false)') assert_equal %w(__FILE__ __LINE__), - R.scan('kw', 'p __FILE__, __LINE__') + scan('kw', 'p __FILE__, __LINE__') assert_equal %w(defined?), - R.scan('kw', 'defined?(Object)') + scan('kw', 'defined?(Object)') end def test_lbrace assert_equal [], - R.scan('lbrace', '') + scan('lbrace', '') assert_equal ['{'], - R.scan('lbrace', '3.times{ }') + scan('lbrace', '3.times{ }') assert_equal ['{'], - R.scan('lbrace', '3.times { }') + scan('lbrace', '3.times { }') assert_equal ['{'], - R.scan('lbrace', '3.times{}') + scan('lbrace', '3.times{}') assert_equal [], - R.scan('lbrace', '"{}"') + scan('lbrace', '"{}"') assert_equal ['{'], - R.scan('lbrace', '{1=>2}') + scan('lbrace', '{1=>2}') end def test_rbrace assert_equal [], - R.scan('rbrace', '') + scan('rbrace', '') assert_equal ['}'], - R.scan('rbrace', '3.times{ }') + scan('rbrace', '3.times{ }') assert_equal ['}'], - R.scan('rbrace', '3.times { }') + scan('rbrace', '3.times { }') assert_equal ['}'], - R.scan('rbrace', '3.times{}') + scan('rbrace', '3.times{}') assert_equal [], - R.scan('rbrace', '"{}"') + scan('rbrace', '"{}"') assert_equal ['}'], - R.scan('rbrace', '{1=>2}') + scan('rbrace', '{1=>2}') end def test_lbracket assert_equal [], - R.scan('lbracket', '') + scan('lbracket', '') assert_equal ['['], - R.scan('lbracket', '[]') + scan('lbracket', '[]') assert_equal ['['], - R.scan('lbracket', 'a[1]') + scan('lbracket', 'a[1]') assert_equal [], - R.scan('lbracket', 'm(%q[])') + scan('lbracket', 'm(%q[])') end def test_rbracket assert_equal [], - R.scan('rbracket', '') + scan('rbracket', '') assert_equal [']'], - R.scan('rbracket', '[]') + scan('rbracket', '[]') assert_equal [']'], - R.scan('rbracket', 'a[1]') + scan('rbracket', 'a[1]') assert_equal [], - R.scan('rbracket', 'm(%q[])') + scan('rbracket', 'm(%q[])') end def test_lparen assert_equal [], - R.scan('lparen', '') + scan('lparen', '') assert_equal ['('], - R.scan('lparen', '()') + scan('lparen', '()') assert_equal ['('], - R.scan('lparen', 'm()') + scan('lparen', 'm()') assert_equal ['('], - R.scan('lparen', 'm (a)') + scan('lparen', 'm (a)') assert_equal [], - R.scan('lparen', '"()"') + scan('lparen', '"()"') assert_equal [], - R.scan('lparen', '"%w()"') + scan('lparen', '"%w()"') end def test_rparen assert_equal [], - R.scan('rparen', '') + scan('rparen', '') assert_equal [')'], - R.scan('rparen', '()') + scan('rparen', '()') assert_equal [')'], - R.scan('rparen', 'm()') + scan('rparen', 'm()') assert_equal [')'], - R.scan('rparen', 'm (a)') + scan('rparen', 'm (a)') assert_equal [], - R.scan('rparen', '"()"') + scan('rparen', '"()"') assert_equal [], - R.scan('rparen', '"%w()"') + scan('rparen', '"%w()"') end def test_op assert_equal [], - R.scan('op', '') + scan('op', '') assert_equal ['|'], - R.scan('op', '1 | 1') + scan('op', '1 | 1') assert_equal ['^'], - R.scan('op', '1 ^ 1') + scan('op', '1 ^ 1') assert_equal ['&'], - R.scan('op', '1 & 1') + scan('op', '1 & 1') assert_equal ['<=>'], - R.scan('op', '1 <=> 1') + scan('op', '1 <=> 1') assert_equal ['=='], - R.scan('op', '1 == 1') + scan('op', '1 == 1') assert_equal ['==='], - R.scan('op', '1 === 1') + scan('op', '1 === 1') assert_equal ['=~'], - R.scan('op', '1 =~ 1') + scan('op', '1 =~ 1') assert_equal ['>'], - R.scan('op', '1 > 1') + scan('op', '1 > 1') assert_equal ['>='], - R.scan('op', '1 >= 1') + scan('op', '1 >= 1') assert_equal ['<'], - R.scan('op', '1 < 1') + scan('op', '1 < 1') assert_equal ['<='], - R.scan('op', '1 <= 1') + scan('op', '1 <= 1') assert_equal ['<<'], - R.scan('op', '1 << 1') + scan('op', '1 << 1') assert_equal ['>>'], - R.scan('op', '1 >> 1') + scan('op', '1 >> 1') assert_equal ['+'], - R.scan('op', '1 + 1') + scan('op', '1 + 1') assert_equal ['-'], - R.scan('op', '1 - 1') + scan('op', '1 - 1') assert_equal ['*'], - R.scan('op', '1 * 1') + scan('op', '1 * 1') assert_equal ['/'], - R.scan('op', '1 / 1') + scan('op', '1 / 1') assert_equal ['%'], - R.scan('op', '1 % 1') + scan('op', '1 % 1') assert_equal ['**'], - R.scan('op', '1 ** 1') + scan('op', '1 ** 1') assert_equal ['~'], - R.scan('op', '~1') + scan('op', '~1') assert_equal ['-'], - R.scan('op', '-a') + scan('op', '-a') assert_equal ['+'], - R.scan('op', '+a') + scan('op', '+a') assert_equal ['[]'], - R.scan('op', ':[]') + scan('op', ':[]') assert_equal ['[]='], - R.scan('op', ':[]=') + scan('op', ':[]=') assert_equal [], - R.scan('op', %q[`make all`]) + scan('op', %q[`make all`]) end def test_symbeg assert_equal [], - R.scan('symbeg', '') + scan('symbeg', '') assert_equal [':'], - R.scan('symbeg', ':sym') + scan('symbeg', ':sym') assert_equal [':'], - R.scan('symbeg', '[1,2,3,:sym]') + scan('symbeg', '[1,2,3,:sym]') assert_equal [], - R.scan('symbeg', '":sym"') + scan('symbeg', '":sym"') assert_equal [], - R.scan('symbeg', 'a ? b : c') + scan('symbeg', 'a ? b : c') end def test_tstring_beg assert_equal [], - R.scan('tstring_beg', '') + scan('tstring_beg', '') assert_equal ['"'], - R.scan('tstring_beg', '"abcdef"') + scan('tstring_beg', '"abcdef"') assert_equal ['%q['], - R.scan('tstring_beg', '%q[abcdef]') + scan('tstring_beg', '%q[abcdef]') assert_equal ['%Q['], - R.scan('tstring_beg', '%Q[abcdef]') + scan('tstring_beg', '%Q[abcdef]') end def test_tstring_content assert_equal [], - R.scan('tstring_content', '') + scan('tstring_content', '') assert_equal ['abcdef'], - R.scan('tstring_content', '"abcdef"') + scan('tstring_content', '"abcdef"') assert_equal ['abcdef'], - R.scan('tstring_content', '%q[abcdef]') + scan('tstring_content', '%q[abcdef]') assert_equal ['abcdef'], - R.scan('tstring_content', '%Q[abcdef]') + scan('tstring_content', '%Q[abcdef]') assert_equal ['abc', 'def'], - R.scan('tstring_content', '"abc#{1}def"') + scan('tstring_content', '"abc#{1}def"') assert_equal ['sym'], - R.scan('tstring_content', ':"sym"') + scan('tstring_content', ':"sym"') end def test_tstring_end assert_equal [], - R.scan('tstring_end', '') + scan('tstring_end', '') assert_equal ['"'], - R.scan('tstring_end', '"abcdef"') + scan('tstring_end', '"abcdef"') assert_equal [']'], - R.scan('tstring_end', '%q[abcdef]') + scan('tstring_end', '%q[abcdef]') assert_equal [']'], - R.scan('tstring_end', '%Q[abcdef]') + scan('tstring_end', '%Q[abcdef]') end def test_regexp_beg assert_equal [], - R.scan('regexp_beg', '') + scan('regexp_beg', '') assert_equal ['/'], - R.scan('regexp_beg', '/re/') + scan('regexp_beg', '/re/') assert_equal ['%r<'], - R.scan('regexp_beg', '%r') + scan('regexp_beg', '%r') assert_equal [], - R.scan('regexp_beg', '5 / 5') + scan('regexp_beg', '5 / 5') end def test_regexp_end assert_equal [], - R.scan('regexp_end', '') + scan('regexp_end', '') assert_equal ['/'], - R.scan('regexp_end', '/re/') + scan('regexp_end', '/re/') assert_equal ['>'], - R.scan('regexp_end', '%r') + scan('regexp_end', '%r') end def test_words_beg assert_equal [], - R.scan('words_beg', '') + scan('words_beg', '') assert_equal ['%W('], - R.scan('words_beg', '%W()') + scan('words_beg', '%W()') assert_equal ['%W('], - R.scan('words_beg', '%W(w w w)') + scan('words_beg', '%W(w w w)') assert_equal ['%W( '], - R.scan('words_beg', '%W( w w w )') + scan('words_beg', '%W( w w w )') end def test_qwords_beg assert_equal [], - R.scan('qwords_beg', '') + scan('qwords_beg', '') assert_equal ['%w('], - R.scan('qwords_beg', '%w()') + scan('qwords_beg', '%w()') assert_equal ['%w('], - R.scan('qwords_beg', '%w(w w w)') + scan('qwords_beg', '%w(w w w)') assert_equal ['%w( '], - R.scan('qwords_beg', '%w( w w w )') + scan('qwords_beg', '%w( w w w )') end # FIXME: Close paren must not present (`words_end' scanner event?). def test_words_sep assert_equal [], - R.scan('words_sep', '') + scan('words_sep', '') assert_equal [')'], - R.scan('words_sep', '%w()') + scan('words_sep', '%w()') assert_equal [' ', ' ', ')'], - R.scan('words_sep', '%w(w w w)') + scan('words_sep', '%w(w w w)') assert_equal [' ', ' ', ' )'], - R.scan('words_sep', '%w( w w w )') + scan('words_sep', '%w( w w w )') assert_equal ["\n", ' ', ' )'], - R.scan('words_sep', "%w( w\nw w )") + scan('words_sep', "%w( w\nw w )") end def test_heredoc_beg assert_equal [], - R.scan('heredoc_beg', '') + scan('heredoc_beg', '') assert_equal ['<