From 214a7f8d49c7b59d06f5e2e3e1a8a3567ab7c570 Mon Sep 17 00:00:00 2001 From: hsbt Date: Tue, 12 Sep 2017 03:42:54 +0000 Subject: [PATCH] Merge rdoc-6.0.0.beta2 from upstream. * This version changed lexer used Ripper from lexer based IRB. see details: https://github.com/ruby/rdoc/pull/512 git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59845 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- lib/rdoc.rb | 4 +- lib/rdoc/constant.rb | 4 +- lib/rdoc/context.rb | 12 + lib/rdoc/markdown.rb | 2 +- lib/rdoc/markup/to_html.rb | 8 +- lib/rdoc/parser/c.rb | 6 +- lib/rdoc/parser/ripper_state_lex.rb | 587 ++++++++++ lib/rdoc/parser/ruby.rb | 814 +++++++------ lib/rdoc/parser/ruby_tools.rb | 32 +- lib/rdoc/rd/block_parser.rb | 120 +- lib/rdoc/rd/inline_parser.rb | 96 +- lib/rdoc/rdoc.gemspec | 10 +- lib/rdoc/ruby_lex.rb | 1521 ------------------------- lib/rdoc/stats/normal.rb | 4 +- lib/rdoc/token_stream.rb | 49 +- test/rdoc/test_rdoc_any_method.rb | 2 +- test/rdoc/test_rdoc_constant.rb | 6 +- test/rdoc/test_rdoc_context.rb | 14 + test/rdoc/test_rdoc_markup_to_html.rb | 3 + test/rdoc/test_rdoc_parser_c.rb | 10 +- test/rdoc/test_rdoc_parser_ruby.rb | 278 +++-- test/rdoc/test_rdoc_ruby_lex.rb | 1095 ------------------ test/rdoc/test_rdoc_token_stream.rb | 22 +- 23 files changed, 1472 insertions(+), 3227 deletions(-) create mode 100644 lib/rdoc/parser/ripper_state_lex.rb delete mode 100644 lib/rdoc/ruby_lex.rb delete mode 100644 test/rdoc/test_rdoc_ruby_lex.rb diff --git a/lib/rdoc.rb b/lib/rdoc.rb index 8482e910f5..3856468421 100644 --- a/lib/rdoc.rb +++ b/lib/rdoc.rb @@ -65,7 +65,7 @@ module RDoc ## # RDoc version you are using - VERSION = '6.0.0.beta1' + VERSION = '6.0.0.beta2' ## # Method visibilities @@ -148,7 +148,7 @@ module RDoc autoload :KNOWN_CLASSES, 'rdoc/known_classes' - autoload :RubyLex, 'rdoc/ruby_lex' + autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex' autoload :RubyToken, 'rdoc/ruby_token' autoload :TokenStream, 'rdoc/token_stream' diff --git a/lib/rdoc/constant.rb b/lib/rdoc/constant.rb index 4fd5c5f10f..7e1e10da72 100644 --- a/lib/rdoc/constant.rb +++ b/lib/rdoc/constant.rb @@ -36,7 +36,7 @@ class RDoc::Constant < RDoc::CodeObject @value = value @is_alias_for = nil - @visibility = nil + @visibility = :public self.comment = comment end @@ -136,7 +136,7 @@ class RDoc::Constant < RDoc::CodeObject initialize array[1], nil, array[5] @full_name = array[2] - @visibility = array[3] + @visibility = array[3] || :public @is_alias_for = array[4] # 5 handled above # 6 handled below diff --git a/lib/rdoc/context.rb b/lib/rdoc/context.rb index 7fd6bb0af9..3294b2561c 100644 --- a/lib/rdoc/context.rb +++ b/lib/rdoc/context.rb @@ -1079,6 +1079,7 @@ class RDoc::Context < RDoc::CodeObject return if [:private, :nodoc].include? min_visibility remove_invisible_in @method_list, min_visibility remove_invisible_in @attributes, min_visibility + remove_invisible_in @constants, min_visibility end ## @@ -1165,6 +1166,17 @@ class RDoc::Context < RDoc::CodeObject end end + ## + # Given an array +names+ of constants, set the visibility of each constant to + # +visibility+ + + def set_constant_visibility_for(names, visibility) + names.each do |name| + constant = @constants_hash[name] or next + constant.visibility = visibility + end + end + ## # Sorts sections alphabetically (default) or in TomDoc fashion (none, # Public, Internal, Deprecated) diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb index b10f5193da..9b433b3ee6 100644 --- a/lib/rdoc/markdown.rb +++ b/lib/rdoc/markdown.rb @@ -15280,7 +15280,7 @@ class RDoc::Markdown self.pos = _save break end - @result = begin; + @result = begin; ref = [:inline, @note_order.length] @footnotes[ref] = paragraph a diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index aa5de7bf66..98ed7926fd 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -200,10 +200,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter content = if verbatim.ruby? or parseable? text then begin - tokens = RDoc::RubyLex.tokenize text, @options + tokens = RDoc::RipperStateLex.parse text klass = ' class="ruby"' - RDoc::TokenStream.to_html tokens + result = RDoc::TokenStream.to_html tokens + result = result + "\n" unless "\n" == result[-1] + result rescue RDoc::RubyLex::Error CGI.escapeHTML text end @@ -212,7 +214,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter end if @options.pipe then - @res << "\n
#{CGI.escapeHTML text}
\n" + @res << "\n
#{CGI.escapeHTML text}\n
\n" else @res << "\n#{content}\n" end diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb index 0e72df2ecd..5c940ab28e 100644 --- a/lib/rdoc/parser/c.rb +++ b/lib/rdoc/parser/c.rb @@ -666,8 +666,7 @@ class RDoc::Parser::C < RDoc::Parser #meth_obj.params = params meth_obj.start_collecting_tokens - tk = RDoc::RubyToken::Token.new nil, 1, 1 - tk.set_text body + tk = { :line_no => 1, :char_no => 1, :text => body } meth_obj.add_token tk meth_obj.comment = comment meth_obj.line = file_content[0, offset].count("\n") + 1 @@ -684,8 +683,7 @@ class RDoc::Parser::C < RDoc::Parser find_modifiers comment, meth_obj meth_obj.start_collecting_tokens - tk = RDoc::RubyToken::Token.new nil, 1, 1 - tk.set_text body + tk = { :line_no => 1, :char_no => 1, :text => body } meth_obj.add_token tk meth_obj.comment = comment meth_obj.line = file_content[0, offset].count("\n") + 1 diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb new file mode 100644 index 0000000000..c9a0f5a21e --- /dev/null +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -0,0 +1,587 @@ +require 'ripper' + +class RDoc::RipperStateLex + EXPR_NONE = 0 + EXPR_BEG = 1 + EXPR_END = 2 + EXPR_ENDARG = 4 + EXPR_ENDFN = 8 + EXPR_ARG = 16 + EXPR_CMDARG = 32 + EXPR_MID = 64 + EXPR_FNAME = 128 + EXPR_DOT = 256 + EXPR_CLASS = 512 + EXPR_LABEL = 1024 + EXPR_LABELED = 2048 + EXPR_FITEM = 4096 + EXPR_VALUE = EXPR_BEG + EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS) + EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG) + EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN) + + class InnerStateLex < Ripper::Filter + attr_accessor :lex_state + + def initialize(code) + @lex_state = EXPR_BEG + @in_fname = false + @continue = false + reset + super(code) + end + + def reset + @command_start = false + @cmd_state = @command_start + end + + def on_nl(tok, data) + case @lex_state + when EXPR_FNAME, EXPR_DOT + @continue = true + else + @continue = false + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_ignored_nl(tok, data) + case @lex_state + when EXPR_FNAME, EXPR_DOT + @continue = true + else + @continue = false + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_op(tok, data) + case tok + when '&', '|', '!', '!=', '!~' + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end + when '<<' + # TODO next token? + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end + when '?' + @lex_state = EXPR_BEG + when '&&', '||', '+=', '-=', '*=', '**=', + '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' + @lex_state = EXPR_BEG + else + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_kw(tok, data) + case tok + when 'class' + @lex_state = EXPR_CLASS + @in_fname = true + when 'def' + @lex_state = EXPR_FNAME + @continue = true + @in_fname = true + when 'if', 'unless', 'while', 'until' + if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if + @lex_state = EXPR_BEG | EXPR_LABEL + else + @lex_state = EXPR_BEG + end + when 'begin' + @lex_state = EXPR_BEG + else + if @lex_state == EXPR_FNAME + @lex_state = EXPR_END + else + @lex_state = EXPR_END + end + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_tstring_beg(tok, data) + @lex_state = EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_tstring_end(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_CHAR(tok, data) + @lex_state = EXPR_END + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_period(tok, data) + @lex_state = EXPR_DOT + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_int(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_float(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rational(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_imaginary(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_symbeg(tok, data) + @lex_state = EXPR_FNAME + @continue = true + @in_fname = true + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + private def on_variables(event, tok, data) + if @in_fname + @lex_state = EXPR_ENDFN + @in_fname = false + @continue = false + elsif @continue + case @lex_state + when EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_ENDFN + @continue = false + end + else + @lex_state = EXPR_CMDARG + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) + end + + def on_ident(tok, data) + on_variables(__method__, tok, data) + end + + def on_ivar(tok, data) + @lex_state = EXPR_END + on_variables(__method__, tok, data) + end + + def on_cvar(tok, data) + @lex_state = EXPR_END + on_variables(__method__, tok, data) + end + + def on_gvar(tok, data) + @lex_state = EXPR_END + on_variables(__method__, tok, data) + end + + def on_backref(tok, data) + @lex_state = EXPR_END + on_variables(__method__, tok, data) + end + + def on_lparen(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rparen(tok, data) + @lex_state = EXPR_ENDFN + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_lbrace(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rbrace(tok, data) + @lex_state = EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_lbracket(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rbracket(tok, data) + @lex_state = EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_const(tok, data) + case @lex_state + when EXPR_FNAME + @lex_state = EXPR_ENDFN + when EXPR_CLASS + @lex_state = EXPR_ARG + else + @lex_state = EXPR_CMDARG + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_sp(tok, data) + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_comma(tok, data) + @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0 + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_comment(tok, data) + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_ignored_sp(tok, data) + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_heredoc_end(tok, data) + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + @lex_state = EXPR_BEG + end + + def on_default(event, tok, data) + reset + @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) + end + + def each(&block) + @callback = block + parse + end + end + + def get_squashed_tk + if @buf.empty? + tk = @inner_lex_enumerator.next + else + tk = @buf.shift + end + case tk[:kind] + when :on_symbeg then + tk = get_symbol_tk(tk) + when :on_tstring_beg then + tk = get_string_tk(tk) + when :on_backtick then + if (EXPR_FNAME & tk[:state]) != 0 + @inner_lex.lex_state = EXPR_ARG + tk[:kind] = :on_ident + tk[:state] = @inner_lex.lex_state + else + tk = get_string_tk(tk) + end + when :on_regexp_beg then + tk = get_regexp_tk(tk) + when :on_embdoc_beg then + tk = get_embdoc_tk(tk) + when :on_heredoc_beg then + @heredoc_queue << retrieve_heredoc_info(tk) + @inner_lex.lex_state = EXPR_END + when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then + unless @heredoc_queue.empty? + get_heredoc_tk(*@heredoc_queue.shift) + end + when :on_words_beg then + tk = get_words_tk(tk) + when :on_qwords_beg then + tk = get_words_tk(tk) + when :on_symbols_beg then + tk = get_words_tk(tk) + when :on_qsymbols_beg then + tk = get_words_tk(tk) + when :on_op then + if '&.' == tk[:text] + tk[:kind] = :on_period + else + tk = get_op_tk(tk) + end + end + tk + end + + private def get_symbol_tk(tk) + is_symbol = true + symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } + if ":'" == tk[:text] or ':"' == tk[:text] + tk1 = get_string_tk(tk) + symbol_tk[:text] = tk1[:text] + symbol_tk[:state] = tk1[:state] + else + case (tk1 = get_squashed_tk)[:kind] + when :on_ident + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_tstring_content + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end + when :on_tstring_end + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_op + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_ivar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_cvar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_gvar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_const + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_kw + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + else + is_symbol = false + tk = tk1 + end + end + if is_symbol + tk = symbol_tk + end + tk + end + + private def get_string_tk(tk) + string = tk[:text] + state = nil + kind = :on_tstring + loop do + inner_str_tk = get_squashed_tk + if inner_str_tk.nil? + break + elsif :on_tstring_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + break + elsif :on_label_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + kind = :on_symbol + break + else + string = string + inner_str_tk[:text] + if :on_embexpr_beg == inner_str_tk[:kind] then + kind = :on_dstring if :on_tstring == kind + end + end + end + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => kind, + :text => string, + :state => state + } + end + + private def get_regexp_tk(tk) + string = tk[:text] + state = nil + loop do + inner_str_tk = get_squashed_tk + if inner_str_tk.nil? + break + elsif :on_regexp_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + break + else + string = string + inner_str_tk[:text] + end + end + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => :on_regexp, + :text => string, + :state => state + } + end + + private def get_embdoc_tk(tk) + string = tk[:text] + until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do + string = string + embdoc_tk[:text] + end + string = string + embdoc_tk[:text] + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => :on_embdoc, + :text => string, + :state => embdoc_tk[:state] + } + end + + private def get_heredoc_tk(heredoc_name, indent) + string = '' + start_tk = nil + prev_tk = nil + until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do + start_tk = tk unless start_tk + if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no] + string = string + (' ' * tk[:char_no]) + end + string = string + tk[:text] + prev_tk = tk + end + start_tk = tk unless start_tk + prev_tk = tk unless prev_tk + @buf.unshift tk # closing heredoc + heredoc_tk = { + :line_no => start_tk[:line_no], + :char_no => start_tk[:char_no], + :kind => :on_heredoc, + :text => string, + :state => prev_tk[:state] + } + @buf.unshift heredoc_tk + end + + private def retrieve_heredoc_info(tk) + name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2') + indent = tk[:text] =~ /\A<<[-~]/ + [name, indent] + end + + private def heredoc_end?(name, indent, tk) + result = false + if :on_heredoc_end == tk[:kind] then + tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, '')) + if name == tk_name + result = true + end + end + result + end + + private def get_words_tk(tk) + string = '' + start_token = tk[:text] + start_quote = tk[:text].rstrip[-1] + line_no = tk[:line_no] + char_no = tk[:char_no] + state = tk[:state] + end_quote = + case start_quote + when ?( then ?) + when ?[ then ?] + when ?{ then ?} + when ?< then ?> + else start_quote + end + end_token = nil + loop do + tk = get_squashed_tk + if tk.nil? + end_token = end_quote + break + elsif :on_tstring_content == tk[:kind] then + string += tk[:text] + elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then + if end_quote == tk[:text].strip then + end_token = tk[:text] + break + else + string += tk[:text] + end + else + string += tk[:text] + end + end + text = "#{start_token}#{string}#{end_token}" + { + :line_no => line_no, + :char_no => char_no, + :kind => :on_dstring, + :text => text, + :state => state + } + end + + private def get_op_tk(tk) + redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~] + if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then + @inner_lex.lex_state = EXPR_ARG + tk[:kind] = :on_ident + tk[:state] = @inner_lex.lex_state + elsif tk[:text] =~ /^[-+]$/ then + tk_ahead = get_squashed_tk + case tk_ahead[:kind] + when :on_int, :on_float, :on_rational, :on_imaginary then + tk[:text] += tk_ahead[:text] + tk[:kind] = tk_ahead[:kind] + tk[:state] = tk_ahead[:state] + else + @buf.unshift tk_ahead + end + end + tk + end + + def initialize(code) + @buf = [] + @heredoc_queue = [] + @inner_lex = InnerStateLex.new(code) + @inner_lex_enumerator = Enumerator.new do |y| + @inner_lex.each do |tk| + y << tk + end + end + end + + def self.parse(code) + lex = self.new(code) + tokens = [] + begin + while tk = lex.get_squashed_tk + tokens.push tk + end + rescue StopIteration + end + tokens + end + + def self.end?(token) + (token[:state] & EXPR_END) + end +end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index e7900c0807..130eca89c7 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -24,6 +24,7 @@ $TOKEN_DEBUG ||= nil # * aliases # * private, public, protected # * private_class_function, public_class_function +# * private_constant, public_constant # * module_function # * attr, attr_reader, attr_writer, attr_accessor # * extra accessors given on the command line @@ -139,11 +140,12 @@ $TOKEN_DEBUG ||= nil # Note that by default, the :method: directive will be ignored if there is a # standard rdocable item following it. +require 'ripper' + class RDoc::Parser::Ruby < RDoc::Parser parse_files_matching(/\.rbw?$/) - include RDoc::RubyToken include RDoc::TokenStream include RDoc::Parser::RubyTools @@ -163,10 +165,20 @@ class RDoc::Parser::Ruby < RDoc::Parser def initialize(top_level, file_name, content, options, stats) super + if /\t/ =~ content then + tab_width = @options.tab_width + content = content.split(/\n/).map do |line| + 1 while line.gsub!(/\t+/) { + ' ' * (tab_width*$&.length - $`.length % tab_width) + } && $~ + line + end.join("\n") + end + @size = 0 @token_listeners = nil - @scanner = RDoc::RubyLex.new content, @options - @scanner.exception_on_syntax_error = false + @scanner = RDoc::RipperStateLex.parse(content) + @scanner_point = 0 @prev_seek = nil @markup = @options.markup @track_visibility = :nodoc != @options.visibility @@ -175,6 +187,10 @@ class RDoc::Parser::Ruby < RDoc::Parser reset end + def tk_nl?(tk) + :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] + end + ## # Retrieves the read token stream and replaces +pattern+ with +replacement+ # using gsub. If the result is only a ";" returns an empty string. @@ -194,7 +210,7 @@ class RDoc::Parser::Ruby < RDoc::Parser # methods. def get_visibility_information tk, single # :nodoc: - vis_type = tk.name + vis_type = tk[:text] singleton = single == SINGLE vis = @@ -226,27 +242,27 @@ class RDoc::Parser::Ruby < RDoc::Parser comment = '' comment.force_encoding @encoding if @encoding first_line = true - first_comment_tk_class = nil + first_comment_tk_kind = nil tk = get_tk - while TkCOMMENT === tk - if first_line and tk.text =~ /\A#!/ then + while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) + if first_line and tk[:text] =~ /\A#!/ then skip_tkspace tk = get_tk - elsif first_line and tk.text =~ /\A#\s*-\*-/ then + elsif first_line and tk[:text] =~ /\A#\s*-\*-/ then first_line = false skip_tkspace tk = get_tk else - break if first_comment_tk_class and not first_comment_tk_class === tk - first_comment_tk_class = tk.class + break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind] + first_comment_tk_kind = tk[:kind] first_line = false - comment << tk.text << "\n" + comment << tk[:text] tk = get_tk - if TkNL === tk then + if :on_nl === tk then skip_tkspace false tk = get_tk end @@ -262,7 +278,6 @@ class RDoc::Parser::Ruby < RDoc::Parser # Consumes trailing whitespace from the token stream def consume_trailing_spaces # :nodoc: - get_tkread skip_tkspace false end @@ -309,10 +324,9 @@ class RDoc::Parser::Ruby < RDoc::Parser def get_bool skip_tkspace tk = get_tk - case tk - when TkTRUE + if :on_kw == tk[:kind] && 'true' == tk[:text] true - when TkFALSE, TkNIL + elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text]) false else unget_tk tk @@ -331,24 +345,24 @@ class RDoc::Parser::Ruby < RDoc::Parser given_name = '' # class ::A -> A is in the top level - case name_t - when TkCOLON2, TkCOLON3 then # bug + if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug name_t = get_tk container = @top_level given_name << '::' end skip_tkspace false - given_name << name_t.name + given_name << name_t[:text] - while TkCOLON2 === peek_tk do + is_self = name_t[:kind] == :on_op && name_t[:text] == '<<' + while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do prev_container = container - container = container.find_module_named name_t.name + container = container.find_module_named name_t[:text] container ||= if ignore_constants then RDoc::Context.new else - c = prev_container.add_module RDoc::NormalModule, name_t.name + c = prev_container.add_module RDoc::NormalModule, name_t[:text] c.ignore unless prev_container.document_children @top_level.add_to_classes_or_modules c c @@ -359,7 +373,10 @@ class RDoc::Parser::Ruby < RDoc::Parser get_tk skip_tkspace false name_t = get_tk - given_name << '::' << name_t.name + unless :on_const == name_t[:kind] || :on_ident == name_t[:kind] + raise RDoc::Error, "Invalid class or module definition: #{given_name}" + end + given_name << '::' << name_t[:text] end skip_tkspace false @@ -371,9 +388,13 @@ class RDoc::Parser::Ruby < RDoc::Parser # Return a superclass, which can be either a constant of an expression def get_class_specification - case peek_tk - when TkSELF then return 'self' - when TkGVAR then return '' + tk = peek_tk + if tk.nil? + return '' + elsif :on_kw == tk[:kind] && 'self' == tk[:text] + return 'self' + elsif :on_gvar == tk[:kind] + return '' end res = get_constant @@ -383,9 +404,10 @@ class RDoc::Parser::Ruby < RDoc::Parser get_tkread # empty out read buffer tk = get_tk + return res unless tk - case tk - when TkNL, TkCOMMENT, TkSEMICOLON then + case tk[:kind] + when :on_nl, :on_comment, :on_embdoc, :on_semicolon then unget_tk(tk) return res end @@ -403,8 +425,8 @@ class RDoc::Parser::Ruby < RDoc::Parser skip_tkspace false tk = get_tk - while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do - res += tk.name + while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do + res += tk[:text] tk = get_tk end @@ -420,7 +442,7 @@ class RDoc::Parser::Ruby < RDoc::Parser nest = 0 - while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do + while :on_lparen == (tk = peek_tk)[:kind] do get_tk skip_tkspace nest += 1 @@ -431,7 +453,7 @@ class RDoc::Parser::Ruby < RDoc::Parser while nest > 0 skip_tkspace tk = get_tk - nest -= 1 if TkRPAREN === tk + nest -= 1 if :on_rparen == tk[:kind] end name @@ -446,13 +468,19 @@ class RDoc::Parser::Ruby < RDoc::Parser # won't catch all cases (such as "a = yield + 1" def get_end_token tk # :nodoc: - case tk - when TkLPAREN, TkfLPAREN - TkRPAREN - when TkRPAREN + case tk[:kind] + when :on_lparen + { + :kind => :on_rparen, + :text => ')' + } + when :on_rparen nil else - TkNL + { + :kind => :on_nl, + :text => "\n" + } end end @@ -461,11 +489,11 @@ class RDoc::Parser::Ruby < RDoc::Parser def get_method_container container, name_t # :nodoc: prev_container = container - container = container.find_module_named(name_t.name) + container = container.find_module_named(name_t[:text]) unless container then constant = prev_container.constants.find do |const| - const.name == name_t.name + const.name == name_t[:text] end if constant then @@ -476,21 +504,21 @@ class RDoc::Parser::Ruby < RDoc::Parser unless container then # TODO seems broken, should starting at Object in @store - obj = name_t.name.split("::").inject(Object) do |state, item| + obj = name_t[:text].split("::").inject(Object) do |state, item| state.const_get(item) end rescue nil type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule unless [Class, Module].include?(obj.class) then - warn("Couldn't find #{name_t.name}. Assuming it's a module") + warn("Couldn't find #{name_t[:text]}. Assuming it's a module") end if type == RDoc::NormalClass then sclass = obj.superclass ? obj.superclass.name : nil - container = prev_container.add_class type, name_t.name, sclass + container = prev_container.add_class type, name_t[:text], sclass else - container = prev_container.add_module type, name_t.name + container = prev_container.add_module type, name_t[:text] end record_location container @@ -504,32 +532,26 @@ class RDoc::Parser::Ruby < RDoc::Parser def get_symbol_or_name tk = get_tk - case tk - when TkSYMBOL then - text = tk.text.sub(/^:/, '') + case tk[:kind] + when :on_symbol then + text = tk[:text].sub(/^:/, '') - if TkASSIGN === peek_tk then + next_tk = peek_tk + if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then get_tk text << '=' end text - when TkId, TkOp then - tk.name - when TkAMPER, - TkDSTRING, - TkSTAR, - TkSTRING then - tk.text + when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then + tk[:text] + when :on_tstring, :on_dstring then + tk[:text][1..-2] else raise RDoc::Error, "Name or symbol expected (got #{tk})" end end - def stop_at_EXPR_END # :nodoc: - @scanner.lex_state == :EXPR_END || !@scanner.continue - end - ## # Marks containers between +container+ and +ancestor+ as ignored @@ -570,7 +592,8 @@ class RDoc::Parser::Ruby < RDoc::Parser def make_message message prefix = "#{@file_name}:" - prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner + tk = peek_tk + prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk "#{prefix} #{message}" end @@ -589,7 +612,7 @@ class RDoc::Parser::Ruby < RDoc::Parser # +comment+. def parse_attr(context, single, tk, comment) - line_no = tk.line_no + line_no = tk[:line_no] args = parse_symbol_arg 1 if args.size > 0 then @@ -598,7 +621,7 @@ class RDoc::Parser::Ruby < RDoc::Parser skip_tkspace false tk = get_tk - if TkCOMMA === tk then + if :on_comma == tk[:kind] then rw = "RW" if get_bool else unget_tk tk @@ -618,7 +641,7 @@ class RDoc::Parser::Ruby < RDoc::Parser # comment for each to +comment+. def parse_attr_accessor(context, single, tk, comment) - line_no = tk.line_no + line_no = tk[:line_no] args = parse_symbol_arg rw = "?" @@ -629,7 +652,7 @@ class RDoc::Parser::Ruby < RDoc::Parser # and add found items appropriately but here we do not. I'm not sure why. return if @track_visibility and not tmp.document_self - case tk.name + case tk[:text] when "attr_reader" then rw = "R" when "attr_writer" then rw = "W" when "attr_accessor" then rw = "RW" @@ -647,21 +670,19 @@ class RDoc::Parser::Ruby < RDoc::Parser # Parses an +alias+ in +context+ with +comment+ def parse_alias(context, single, tk, comment) - line_no = tk.line_no + line_no = tk[:line_no] skip_tkspace - if TkLPAREN === peek_tk then + if :on_lparen === peek_tk[:kind] then get_tk skip_tkspace end new_name = get_symbol_or_name - @scanner.lex_state = :EXPR_FNAME - skip_tkspace - if TkCOMMA === peek_tk then + if :on_comma === peek_tk[:kind] then get_tk skip_tkspace end @@ -688,34 +709,38 @@ class RDoc::Parser::Ruby < RDoc::Parser # Extracts call parameters from the token stream. def parse_call_parameters(tk) - end_token = case tk - when TkLPAREN, TkfLPAREN - TkRPAREN - when TkRPAREN + end_token = case tk[:kind] + when :on_lparen + :on_rparen + when :on_rparen return "" else - TkNL + :on_nl end nest = 0 loop do - case tk - when TkSEMICOLON + break if tk.nil? + case tk[:kind] + when :on_semicolon break - when TkLPAREN, TkfLPAREN + when :on_lparen nest += 1 when end_token - if end_token == TkRPAREN + if end_token == :on_rparen nest -= 1 - break if @scanner.lex_state == :EXPR_END and nest <= 0 + break if RDoc::RipperStateLex.end?(tk) and nest <= 0 else - break unless @scanner.continue + break if RDoc::RipperStateLex.end?(tk) end - when TkCOMMENT, TkASSIGN, TkOPASGN + when :on_comment, :on_embdoc unget_tk(tk) break - when nil then - break + when :on_op + if tk[:text] =~ /^(.{1,2})?=$/ + unget_tk(tk) + break + end end tk = get_tk end @@ -727,28 +752,27 @@ class RDoc::Parser::Ruby < RDoc::Parser # Parses a class in +context+ with +comment+ def parse_class container, single, tk, comment - line_no = tk.line_no + line_no = tk[:line_no] declaration_context = container container, name_t, given_name = get_class_or_module container - cls = - case name_t - when TkCONSTANT - parse_class_regular container, declaration_context, single, - name_t, given_name, comment - when TkLSHFT - case name = get_class_specification - when 'self', container.name - parse_statements container, SINGLE - return # don't update line - else - parse_class_singleton container, name, comment - end + if name_t[:kind] == :on_const + cls = parse_class_regular container, declaration_context, single, + name_t, given_name, comment + elsif name_t[:kind] == :on_op && name_t[:text] == '<<' + case name = get_class_specification + when 'self', container.name + read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS + parse_statements container, SINGLE + return # don't update line else - warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}" - return + cls = parse_class_singleton container, name, comment end + else + warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}" + return + end cls.line = line_no @@ -770,7 +794,8 @@ class RDoc::Parser::Ruby < RDoc::Parser given_name = $' end - if TkLT === peek_tk then + tk = peek_tk + if tk[:kind] == :on_op && tk[:text] == '<' then get_tk skip_tkspace superclass = get_class_specification @@ -840,40 +865,52 @@ class RDoc::Parser::Ruby < RDoc::Parser # true, no found constants will be added to RDoc. def parse_constant container, tk, comment, ignore_constants = false - line_no = tk.line_no + line_no = tk[:line_no] - name = tk.name + name = tk[:text] skip_tkspace false return unless name =~ /^\w+$/ eq_tk = get_tk - if TkCOLON2 === eq_tk then + if :on_op == eq_tk[:kind] && '::' == eq_tk[:text] then unget_tk eq_tk unget_tk tk container, name_t, = get_class_or_module container, ignore_constants - name = name_t.name + name = name_t[:text] eq_tk = get_tk end - unless TkASSIGN === eq_tk then - unget_tk eq_tk - return false + is_array_or_hash = false + if eq_tk && :on_lbracket == eq_tk[:kind] + nest = 1 + while bracket_tk = get_tk + case bracket_tk[:kind] + when :on_lbracket + nest += 1 + when :on_rbracket + nest -= 1 + break if nest == 0 + end + end + skip_tkspace false + eq_tk = get_tk + is_array_or_hash = true end - if TkGT === peek_tk then + unless eq_tk && :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then unget_tk eq_tk - return + return false end value = '' con = RDoc::Constant.new name, value, comment - body = parse_constant_body container, con + body = parse_constant_body container, con, is_array_or_hash return unless body @@ -882,13 +919,15 @@ class RDoc::Parser::Ruby < RDoc::Parser con.line = line_no read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS + return if is_array_or_hash + @stats.add_constant con container.add_constant con true end - def parse_constant_body container, constant # :nodoc: + def parse_constant_body container, constant, is_array_or_hash # :nodoc: nest = 0 rhs_name = '' @@ -896,44 +935,51 @@ class RDoc::Parser::Ruby < RDoc::Parser tk = get_tk + body = nil loop do - case tk - when TkSEMICOLON then + break if tk.nil? + if :on_semicolon == tk[:kind] then break if nest <= 0 - when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK, - TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then + elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then nest += 1 - when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then + elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then + nest += 1 + elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then + if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + nest += 1 + end + elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) || + (:on_kw == tk[:kind] && 'end' == tk[:text]) then nest -= 1 - when TkCOMMENT then - if nest <= 0 and stop_at_EXPR_END then - unget_tk tk + elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then + unget_tk tk + if nest <= 0 and RDoc::RipperStateLex.end?(tk) then + body = get_tkread_clean(/^[ \t]+/, '') + read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS break else - unget_tk tk read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS end - when TkCONSTANT then - rhs_name << tk.name + elsif :on_const == tk[:kind] then + rhs_name << tk[:text] - if nest <= 0 and TkNL === peek_tk then - create_module_alias container, constant, rhs_name + next_tk = peek_tk + if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then + create_module_alias container, constant, rhs_name unless is_array_or_hash break end - when TkNL then - if nest <= 0 and stop_at_EXPR_END then + elsif :on_nl == tk[:kind] then + if nest <= 0 and RDoc::RipperStateLex.end?(tk) then unget_tk tk break end - when TkCOLON2, TkCOLON3 then + elsif :on_op == tk[:kind] && '::' == tk[:text] rhs_name << '::' - when nil then - break end tk = get_tk end - get_tkread_clean(/^[ \t]+/, '') + body ? body : get_tkread_clean(/^[ \t]+/, '') end ## @@ -942,8 +988,8 @@ class RDoc::Parser::Ruby < RDoc::Parser def parse_comment container, tk, comment return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc' - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] text = comment.text @@ -988,12 +1034,11 @@ class RDoc::Parser::Ruby < RDoc::Parser record_location meth meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - position_comment = TkCOMMENT.new 0, line_no, 1 - position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [position_comment, newline, indent] meth.params = if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then @@ -1022,8 +1067,8 @@ class RDoc::Parser::Ruby < RDoc::Parser def parse_comment_tomdoc container, tk, comment return unless signature = RDoc::TomDoc.signature(comment) - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] name, = signature.split %r%[ \(]%, 2 @@ -1032,12 +1077,11 @@ class RDoc::Parser::Ruby < RDoc::Parser meth.line = line_no meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - position_comment = TkCOMMENT.new 0, line_no, 1 - position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [position_comment, newline, indent] meth.call_seq = signature @@ -1067,7 +1111,7 @@ class RDoc::Parser::Ruby < RDoc::Parser record_location obj end - return unless TkCOMMA === peek_tk + return if peek_tk.nil? || :on_comma != peek_tk[:kind] get_tk end @@ -1079,11 +1123,14 @@ class RDoc::Parser::Ruby < RDoc::Parser # Returns true if the comment was not consumed. def parse_identifier container, single, tk, comment # :nodoc: - case tk.name + case tk[:text] when 'private', 'protected', 'public', 'private_class_method', 'public_class_method', 'module_function' then parse_visibility container, single, tk return true + when 'private_constant', 'public_constant' + parse_constant_visibility container, single, tk + return true when 'attr' then parse_attr container, single, tk, comment when /^attr_(reader|writer|accessor)$/ then @@ -1172,8 +1219,8 @@ class RDoc::Parser::Ruby < RDoc::Parser # Parses a meta-programmed method def parse_meta_method(container, single, tk, comment) - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] start_collecting_tokens add_token tk @@ -1195,12 +1242,11 @@ class RDoc::Parser::Ruby < RDoc::Parser remove_token_listener self meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - position_comment = TkCOMMENT.new 0, line_no, 1 - position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [position_comment, newline, indent] meth.add_tokens @token_stream parse_meta_method_params container, single, meth, tk, comment @@ -1224,17 +1270,16 @@ class RDoc::Parser::Ruby < RDoc::Parser name_t = get_tk - case name_t - when TkSYMBOL then - name_t.text[1..-1] - when TkSTRING then - name_t.value[1..-2] - when TkASSIGN then # ignore + if :on_symbol == name_t[:kind] then + name_t[:text][1..-1] + elsif :on_tstring == name_t[:kind] then + name_t[:text][1..-2] + elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore remove_token_listener self nil else - warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'" + warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'" 'unknown' end end @@ -1254,14 +1299,13 @@ class RDoc::Parser::Ruby < RDoc::Parser last_tk = tk while tk = get_tk do - case tk - when TkSEMICOLON then + if :on_semicolon == tk[:kind] then break - when TkNL then - break unless last_tk and TkCOMMA === last_tk - when TkSPACE then + elsif :on_nl == tk[:kind] then + break unless last_tk and :on_comma == last_tk[:kind] + elsif :on_sp == tk[:kind] then # expression continues - when TkDO then + elsif :on_kw == tk[:kind] && 'do' == tk[:text] then parse_statements container, single, meth break else @@ -1278,8 +1322,8 @@ class RDoc::Parser::Ruby < RDoc::Parser singleton = nil added_container = false name = nil - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] start_collecting_tokens add_token tk @@ -1299,12 +1343,11 @@ class RDoc::Parser::Ruby < RDoc::Parser meth.line = line_no meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - token = TkCOMMENT.new 0, line_no, 1 - token.set_text "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [token, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + token = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + token[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [token, newline, indent] meth.add_tokens @token_stream parse_method_params_and_body container, single, meth, added_container @@ -1325,7 +1368,6 @@ class RDoc::Parser::Ruby < RDoc::Parser def parse_method_params_and_body container, single, meth, added_container token_listener meth do - @scanner.continue = false parse_method_parameters meth if meth.document_self or not @track_visibility then @@ -1368,15 +1410,13 @@ class RDoc::Parser::Ruby < RDoc::Parser # it is a singleton or regular method. def parse_method_name container # :nodoc: - @scanner.lex_state = :EXPR_FNAME - skip_tkspace name_t = get_tk - back_tk = skip_tkspace + back_tk = skip_tkspace(false) singleton = false - case dot = get_tk - when TkDOT, TkCOLON2 then + dot = get_tk + if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then singleton = true name, container = parse_method_name_singleton container, name_t @@ -1397,16 +1437,15 @@ class RDoc::Parser::Ruby < RDoc::Parser # is parsed from the token stream for a regular method. def parse_method_name_regular container, name_t # :nodoc: - case name_t - when TkSTAR, TkAMPER then - name_t.text + if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then + name_t[:text] else - unless name_t.respond_to? :name then + unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then warn "expected method name token, . or ::, got #{name_t.inspect}" skip_method container return end - name_t.name + name_t[:text] end end @@ -1416,47 +1455,43 @@ class RDoc::Parser::Ruby < RDoc::Parser # for a singleton method. def parse_method_name_singleton container, name_t # :nodoc: - @scanner.lex_state = :EXPR_FNAME skip_tkspace name_t2 = get_tk - name = - case name_t - when TkSELF, TkMOD then - case name_t2 - # NOTE: work around '[' being consumed early and not being re-tokenized - # as a TkAREF - when TkfLBRACK then - get_tk - '[]' - else - name_t2.name - end - when TkCONSTANT then - name = name_t2.name - - container = get_method_container container, name_t - - return unless container - - name - when TkIDENTIFIER, TkIVAR, TkGVAR then - parse_method_dummy container - - nil - when TkTRUE, TkFALSE, TkNIL then - klass_name = "#{name_t.name.capitalize}Class" - container = @store.find_class_named klass_name - container ||= @top_level.add_class RDoc::NormalClass, klass_name - - name_t2.name + if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then + # NOTE: work around '[' being consumed early and not being re-tokenized + # as a TkAREF + if :on_lbracket == name_t2[:kind] + get_tk + name = '[]' else - warn "unexpected method name token #{name_t.inspect}" - # break - skip_method container - - nil + name = name_t2[:text] end + elsif :on_const == name_t[:kind] then + name = name_t2[:text] + + container = get_method_container container, name_t + + return unless container + + name + elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then + parse_method_dummy container + + name = nil + elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then + klass_name = "#{name_t[:text].capitalize}Class" + container = @store.find_class_named klass_name + container ||= @top_level.add_class RDoc::NormalClass, klass_name + + name = name_t2[:text] + else + warn "unexpected method name token #{name_t.inspect}" + # break + skip_method container + + name = nil + end return name, container end @@ -1472,43 +1507,49 @@ class RDoc::Parser::Ruby < RDoc::Parser return '' unless end_token nest = 0 + continue = false - loop do - case tk - when TkSEMICOLON then + while tk != nil do + case tk[:kind] + when :on_semicolon then break if nest == 0 - when TkLBRACE, TkfLBRACE then + when :on_lbrace then nest += 1 - when TkRBRACE then + when :on_rbrace then nest -= 1 if nest <= 0 # we might have a.each { |i| yield i } unget_tk(tk) if nest < 0 break end - when TkLPAREN, TkfLPAREN then + when :on_lparen then nest += 1 - when end_token then - if end_token == TkRPAREN + when end_token[:kind] then + if end_token[:kind] == :on_rparen nest -= 1 break if nest <= 0 else - break unless @scanner.continue + break end - when TkRPAREN then + when :on_rparen then nest -= 1 - when method && method.block_params.nil? && TkCOMMENT then - unget_tk tk - read_documentation_modifiers method, modifiers + when :on_comment, :on_embdoc then @read.pop - when TkCOMMENT then - @read.pop - when nil then - break + if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and + (!continue or (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) != 0) then + if method && method.block_params.nil? then + unget_tk tk + read_documentation_modifiers method, modifiers + end + break if !continue and nest <= 0 + end + when :on_comma then + continue = true + when :on_ident then + continue = false if continue end tk = get_tk end - @scanner.first_in_method_statement = true get_tkread_clean(/\s+/, ' ') end @@ -1539,7 +1580,7 @@ class RDoc::Parser::Ruby < RDoc::Parser def parse_module container, single, tk, comment container, name_t, = get_class_or_module container - name = name_t.name + name = name_t[:text] mod = container.add_module RDoc::NormalModule, name mod.ignore unless container.document_children @@ -1562,12 +1603,12 @@ class RDoc::Parser::Ruby < RDoc::Parser skip_tkspace_comment tk = get_tk - if TkLPAREN === tk then + if :on_lparen == tk[:kind] then skip_tkspace_comment tk = get_tk end - name = tk.text if TkSTRING === tk + name = tk[:text][1..-2] if :on_tstring == tk[:kind] if name then @top_level.add_require RDoc::Require.new(name, comment) @@ -1582,14 +1623,15 @@ class RDoc::Parser::Ruby < RDoc::Parser def parse_rescue skip_tkspace false + nest = 0 while tk = get_tk - case tk - when TkNL, TkSEMICOLON then + case tk[:kind] + when :on_nl, :on_semicolon, :on_comment then break - when TkCOMMA then + when :on_comma then skip_tkspace false - get_tk if TkNL === peek_tk + get_tk if :on_nl == peek_tk[:kind] end skip_tkspace false @@ -1613,14 +1655,32 @@ class RDoc::Parser::Ruby < RDoc::Parser keep_comment = false try_parse_comment = false - non_comment_seen = true unless TkCOMMENT === tk + non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) - case tk - when TkNL then - skip_tkspace - tk = get_tk + case tk[:kind] + when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then + if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] + skip_tkspace + tk = get_tk + else + past_tokens = @read.size > 1 ? @read[0..-2] : [] + nl_position = 0 + past_tokens.reverse.each_with_index do |read_tk, i| + if read_tk =~ /^\n$/ then + nl_position = (past_tokens.size - 1) - i + break + elsif read_tk =~ /^#.*\n$/ then + nl_position = ((past_tokens.size - 1) - i) + 1 + break + end + end + comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ } + unless comment_only_line then + tk = get_tk + end + end - if TkCOMMENT === tk then + if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then if non_comment_seen then # Look for RDoc in a comment about to be thrown away non_comment_seen = parse_comment container, tk, comment unless @@ -1630,15 +1690,14 @@ class RDoc::Parser::Ruby < RDoc::Parser comment.force_encoding @encoding if @encoding end - while TkCOMMENT === tk do - comment << tk.text << "\n" + while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do + comment << tk[:text] + comment << "\n" unless "\n" == tk[:text].chars.to_a.last - tk = get_tk - - if TkNL === tk then + if tk[:text].size > 1 && "\n" == tk[:text].chars.to_a.last then skip_tkspace false # leading spaces - tk = get_tk end + tk = get_tk end comment = new_comment comment @@ -1661,59 +1720,76 @@ class RDoc::Parser::Ruby < RDoc::Parser keep_comment = true container.current_line_visibility = nil - when TkCLASS then - parse_class container, single, tk, comment + when :on_kw then + case tk[:text] + when 'class' then + parse_class container, single, tk, comment - when TkMODULE then - parse_module container, single, tk, comment + when 'module' then + parse_module container, single, tk, comment - when TkDEF then - parse_method container, single, tk, comment + when 'def' then + parse_method container, single, tk, comment - when TkCONSTANT then + when 'alias' then + parse_alias container, single, tk, comment unless current_method + + when 'yield' then + if current_method.nil? then + warn "Warning: yield outside of method" if container.document_self + else + parse_yield container, single, tk, current_method + end + + when 'until', 'while' then + if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + nest += 1 + skip_optional_do_after_expression + end + + # Until and While can have a 'do', which shouldn't increase the nesting. + # We can't solve the general case, but we can handle most occurrences by + # ignoring a do at the end of a line. + + # 'for' is trickier + when 'for' then + nest += 1 + skip_for_variable + skip_optional_do_after_expression + + when 'case', 'do', 'if', 'unless', 'begin' then + if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + nest += 1 + end + + when 'super' then + current_method.calls_super = true if current_method + + when 'rescue' then + parse_rescue + + when 'end' then + nest -= 1 + if nest == 0 then + container.ongoing_visibility = save_visibility + + parse_comment container, tk, comment unless comment.empty? + + return + end + end + + when :on_const then unless parse_constant container, tk, comment, current_method then try_parse_comment = true end - when TkALIAS then - parse_alias container, single, tk, comment unless current_method - - when TkYIELD then - if current_method.nil? then - warn "Warning: yield outside of method" if container.document_self - else - parse_yield container, single, tk, current_method - end - - # Until and While can have a 'do', which shouldn't increase the nesting. - # We can't solve the general case, but we can handle most occurrences by - # ignoring a do at the end of a line. - - when TkUNTIL, TkWHILE then - nest += 1 - skip_optional_do_after_expression - - # 'for' is trickier - when TkFOR then - nest += 1 - skip_for_variable - skip_optional_do_after_expression - - when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then - nest += 1 - - when TkSUPER then - current_method.calls_super = true if current_method - - when TkRESCUE then - parse_rescue - - when TkIDENTIFIER then + when :on_ident then if nest == 1 and current_method.nil? then keep_comment = parse_identifier container, single, tk, comment end - case tk.name + case tk[:text] when "require" then parse_require container, comment when "include" then @@ -1722,15 +1798,6 @@ class RDoc::Parser::Ruby < RDoc::Parser parse_extend_or_include RDoc::Extend, container, comment end - when TkEND then - nest -= 1 - if nest == 0 then - container.ongoing_visibility = save_visibility - - parse_comment container, tk, comment unless comment.empty? - - return - end else try_parse_comment = nest == 1 end @@ -1762,8 +1829,8 @@ class RDoc::Parser::Ruby < RDoc::Parser def parse_symbol_arg(no = nil) skip_tkspace_comment - case tk = get_tk - when TkLPAREN + tk = get_tk + if tk[:kind] == :on_lparen parse_symbol_arg_paren no else parse_symbol_arg_space no, tk @@ -1785,10 +1852,10 @@ class RDoc::Parser::Ruby < RDoc::Parser end skip_tkspace_comment - case tk2 = get_tk - when TkRPAREN + case (tk2 = get_tk)[:kind] + when :on_rparen break - when TkCOMMA + when :on_comma else warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC break @@ -1815,7 +1882,7 @@ class RDoc::Parser::Ruby < RDoc::Parser skip_tkspace false tk1 = get_tk - unless TkCOMMA === tk1 then + if tk1.nil? || :on_comma != tk1[:kind] then unget_tk tk1 break end @@ -1834,12 +1901,12 @@ class RDoc::Parser::Ruby < RDoc::Parser # Returns symbol text from the next token def parse_symbol_in_arg - case tk = get_tk - when TkSYMBOL - tk.text.sub(/^:/, '') - when TkSTRING - eval @read[-1] - when TkDSTRING, TkIDENTIFIER then + tk = get_tk + if :on_symbol == tk[:kind] then + tk[:text].sub(/^:/, '') + elsif :on_tstring == tk[:kind] then + tk[:text][1..-2] + elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then nil # ignore else warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC @@ -1873,21 +1940,37 @@ class RDoc::Parser::Ruby < RDoc::Parser skip_tkspace_comment false - case peek_tk - # Ryan Davis suggested the extension to ignore modifiers, because he - # often writes - # - # protected unless $TESTING - # - when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then + ptk = peek_tk + # Ryan Davis suggested the extension to ignore modifiers, because he + # often writes + # + # protected unless $TESTING + # + if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then container.ongoing_visibility = vis - when TkDEF + elsif :on_kw == ptk[:kind] && 'def' == ptk[:text] container.current_line_visibility = vis else update_visibility container, vis_type, vis, singleton end end + ## + # Parses a Module#private_constant or Module#public_constant call from +tk+. + + def parse_constant_visibility(container, single, tk) + args = parse_symbol_arg + case tk[:text] + when 'private_constant' + vis = :private + when 'public_constant' + vis = :public + else + raise RDoc::Error, 'Unreachable' + end + container.set_constant_visibility_for args, vis + end + ## # Determines the block parameter for +context+ @@ -1895,7 +1978,6 @@ class RDoc::Parser::Ruby < RDoc::Parser return if method.block_params get_tkread - @scanner.continue = false method.block_params = parse_method_or_yield_parameters end @@ -1919,11 +2001,10 @@ class RDoc::Parser::Ruby < RDoc::Parser while tk = get_tk do tokens << tk - case tk - when TkNL, TkDEF then + if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then return - when TkCOMMENT then - return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/ + elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then + return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/ directive = $1.downcase @@ -1933,7 +2014,7 @@ class RDoc::Parser::Ruby < RDoc::Parser end end ensure - unless tokens.length == 1 and TkCOMMENT === tokens.first then + unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then tokens.reverse_each do |token| unget_tk token end @@ -1947,6 +2028,7 @@ class RDoc::Parser::Ruby < RDoc::Parser # See also RDoc::Markup::PreProcess#handle_directive def read_documentation_modifiers context, allowed + skip_tkspace(false) directive, value = read_directive allowed return unless directive @@ -1995,27 +2077,25 @@ class RDoc::Parser::Ruby < RDoc::Parser rescue StandardError => e bytes = '' - 20.times do @scanner.ungetc end - count = 0 - 60.times do |i| - count = i - byte = @scanner.getc - break unless byte - bytes << byte + if @scanner_point >= @scanner.size + now_line_no = @scanner[@scanner.size - 1][:line_no] + else + now_line_no = peek_tk[:line_no] end - count -= 20 - count.times do @scanner.ungetc end $stderr.puts <<-EOF -#{self.class} failure around line #{@scanner.line_no} of +#{self.class} failure around line #{now_line_no} of #{@file_name} EOF unless bytes.empty? then $stderr.puts - $stderr.puts bytes.inspect + now_line_no = peek_tk[:line_no] + start_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no } + end_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 } - 1 + $stderr.puts @scanner[start_index..end_index].join end raise e @@ -2034,31 +2114,36 @@ class RDoc::Parser::Ruby < RDoc::Parser b_nest = 0 nest = 0 - @scanner.continue = false loop do - case tk - when TkSEMICOLON, TkNL then + break unless tk + case tk[:kind] + when :on_semicolon, :on_nl, :on_ignored_nl then break if b_nest.zero? - when TkLPAREN, TkfLPAREN then + when :on_lparen then nest += 1 - when TkRPAREN then + when :on_rparen then nest -= 1 - when TkBEGIN then - b_nest += 1 - when TkEND then - b_nest -= 1 - when TkDO - break if nest.zero? - when nil then - break + when :on_kw then + case tk[:text] + when 'begin' + b_nest += 1 + when 'end' + b_nest -= 1 + when 'do' + break if nest.zero? + end + when :on_comment, :on_embdoc then + if b_nest.zero? and "\n" == tk[:text][-1] then + break + end end tk = get_tk end skip_tkspace false - get_tk if TkDO === peek_tk + get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text] end ## @@ -2069,7 +2154,7 @@ class RDoc::Parser::Ruby < RDoc::Parser get_tk skip_tkspace false tk = get_tk - unget_tk(tk) unless TkIN === tk + unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text] end ## @@ -2087,7 +2172,8 @@ class RDoc::Parser::Ruby < RDoc::Parser def skip_tkspace_comment(skip_nl = true) loop do skip_tkspace skip_nl - return unless TkCOMMENT === peek_tk + next_tk = peek_tk + return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind]) get_tk end end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index e35b420f30..0a566827a5 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -5,8 +5,6 @@ module RDoc::Parser::RubyTools - include RDoc::RubyToken - ## # Adds a token listener +obj+, but you should probably use token_listener @@ -22,16 +20,25 @@ module RDoc::Parser::RubyTools tk = nil if @tokens.empty? then - tk = @scanner.token - @read.push @scanner.get_readed - puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG + if @scanner_point >= @scanner.size + return nil + else + tk = @scanner[@scanner_point] + @scanner_point += 1 + @read.push tk[:text] + puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG + end else @read.push @unget_read.shift tk = @tokens.shift puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG end - tk = nil if TkEND_OF_SCRIPT === tk + if tk == nil || :on___end__ == tk[:kind] + tk = nil + end + + return nil unless tk # inform any listeners of our shiny new token @token_listeners.each do |obj| @@ -102,19 +109,24 @@ module RDoc::Parser::RubyTools @tokens = [] @unget_read = [] @nest = 0 + @scanner_point = 0 + end + + def tk_nl?(tk) + :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] end ## # Skips whitespace tokens including newlines if +skip_nl+ is true - def skip_tkspace(skip_nl = true) # HACK dup + def skip_tkspace(skip_nl = true) tokens = [] - while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do - tokens.push tk + while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do + tokens.push(tk) end - unget_tk tk + unget_tk(tk) tokens end diff --git a/lib/rdoc/rd/block_parser.rb b/lib/rdoc/rd/block_parser.rb index 3f4941168f..8ac7cad996 100644 --- a/lib/rdoc/rd/block_parser.rb +++ b/lib/rdoc/rd/block_parser.rb @@ -677,54 +677,54 @@ Racc_debug_parser = false # reduce 0 omitted def _reduce_1(val, _values, result) - result = RDoc::Markup::Document.new(*val[0]) + result = RDoc::Markup::Document.new(*val[0]) result end def _reduce_2(val, _values, result) - raise ParseError, "file empty" + raise ParseError, "file empty" result end def _reduce_3(val, _values, result) - result = val[0].concat val[1] + result = val[0].concat val[1] result end def _reduce_4(val, _values, result) - result = val[0] + result = val[0] result end def _reduce_5(val, _values, result) - result = val + result = val result end def _reduce_6(val, _values, result) - result = val + result = val result end # reduce 7 omitted def _reduce_8(val, _values, result) - result = val + result = val result end def _reduce_9(val, _values, result) - result = val + result = val result end def _reduce_10(val, _values, result) - result = [RDoc::Markup::BlankLine.new] + result = [RDoc::Markup::BlankLine.new] result end def _reduce_11(val, _values, result) - result = val[0].parts + result = val[0].parts result end @@ -732,30 +732,30 @@ def _reduce_12(val, _values, result) # val[0] is like [level, title] title = @inline_parser.parse(val[0][1]) result = RDoc::Markup::Heading.new(val[0][0], title) - + result end def _reduce_13(val, _values, result) result = RDoc::Markup::Include.new val[0], @include_path - + result end def _reduce_14(val, _values, result) # val[0] is Array of String result = paragraph val[0] - + result end def _reduce_15(val, _values, result) - result << val[1].rstrip + result << val[1].rstrip result end def _reduce_16(val, _values, result) - result = [val[0].rstrip] + result = [val[0].rstrip] result end @@ -766,7 +766,7 @@ def _reduce_17(val, _values, result) # imform to lexer. @in_verbatim = false - + result end @@ -777,25 +777,25 @@ def _reduce_18(val, _values, result) # imform to lexer. @in_verbatim = false - + result end def _reduce_19(val, _values, result) result << val[1] - + result end def _reduce_20(val, _values, result) result.concat val[2] - + result end def _reduce_21(val, _values, result) result << "\n" - + result end @@ -803,7 +803,7 @@ def _reduce_22(val, _values, result) result = val # inform to lexer. @in_verbatim = true - + result end @@ -817,89 +817,89 @@ end def _reduce_27(val, _values, result) result = val[0] - + result end def _reduce_28(val, _values, result) result = val[1] - + result end def _reduce_29(val, _values, result) result = val[1].push(val[2]) - + result end def _reduce_30(val, _values, result) - result = val[0] << val[1] + result = val[0] << val[1] result end def _reduce_31(val, _values, result) - result = [val[0]] + result = [val[0]] result end def _reduce_32(val, _values, result) result = RDoc::Markup::List.new :BULLET, *val[0] - + result end def _reduce_33(val, _values, result) - result.push(val[1]) + result.push(val[1]) result end def _reduce_34(val, _values, result) - result = val + result = val result end def _reduce_35(val, _values, result) result = RDoc::Markup::ListItem.new nil, val[0], *val[1] - + result end def _reduce_36(val, _values, result) result = RDoc::Markup::List.new :NUMBER, *val[0] - + result end def _reduce_37(val, _values, result) - result.push(val[1]) + result.push(val[1]) result end def _reduce_38(val, _values, result) - result = val + result = val result end def _reduce_39(val, _values, result) result = RDoc::Markup::ListItem.new nil, val[0], *val[1] - + result end def _reduce_40(val, _values, result) result = RDoc::Markup::List.new :NOTE, *val[0] - + result end def _reduce_41(val, _values, result) - result.push(val[1]) + result.push(val[1]) result end def _reduce_42(val, _values, result) - result = val + result = val result end @@ -907,77 +907,77 @@ def _reduce_43(val, _values, result) term = @inline_parser.parse val[0].strip result = RDoc::Markup::ListItem.new term, *val[1] - + result end def _reduce_44(val, _values, result) result = RDoc::Markup::List.new :LABEL, *val[0] - + result end def _reduce_45(val, _values, result) - result.push(val[1]) + result.push(val[1]) result end def _reduce_46(val, _values, result) - result = val + result = val result end def _reduce_47(val, _values, result) result = RDoc::Markup::ListItem.new "#{val[0].strip}", *val[1] - + result end def _reduce_48(val, _values, result) result = [val[1]].concat(val[2]) - + result end def _reduce_49(val, _values, result) result = [val[1]] - + result end def _reduce_50(val, _values, result) result = val[2] - + result end def _reduce_51(val, _values, result) result = [] - + result end def _reduce_52(val, _values, result) - result.concat val[1] + result.concat val[1] result end # reduce 53 omitted def _reduce_54(val, _values, result) - result = val + result = val result end def _reduce_55(val, _values, result) - result = val + result = val result end # reduce 56 omitted def _reduce_57(val, _values, result) - result = [] + result = [] result end @@ -991,58 +991,58 @@ end def _reduce_62(val, _values, result) result = paragraph [val[0]].concat(val[1]) - + result end def _reduce_63(val, _values, result) result = paragraph [val[0]] - + result end def _reduce_64(val, _values, result) result = paragraph [val[0]].concat(val[1]) - + result end def _reduce_65(val, _values, result) result = paragraph [val[0]] - + result end def _reduce_66(val, _values, result) result = [val[0]].concat(val[1]) - + result end def _reduce_67(val, _values, result) - result.concat val[1] + result.concat val[1] result end def _reduce_68(val, _values, result) - result = val[1] + result = val[1] result end def _reduce_69(val, _values, result) - result = val + result = val result end # reduce 70 omitted def _reduce_71(val, _values, result) - result = [] + result = [] result end def _reduce_72(val, _values, result) - result = [] + result = [] result end diff --git a/lib/rdoc/rd/inline_parser.rb b/lib/rdoc/rd/inline_parser.rb index cc63ea6f70..4a3f3ed7b5 100644 --- a/lib/rdoc/rd/inline_parser.rb +++ b/lib/rdoc/rd/inline_parser.rb @@ -732,12 +732,12 @@ Racc_debug_parser = false # reduce 1 omitted def _reduce_2(val, _values, result) - result.append val[1] + result.append val[1] result end def _reduce_3(val, _values, result) - result = val[0] + result = val[0] result end @@ -762,28 +762,28 @@ end def _reduce_13(val, _values, result) content = val[1] result = inline "#{content}", content - + result end def _reduce_14(val, _values, result) content = val[1] result = inline "#{content}", content - + result end def _reduce_15(val, _values, result) content = val[1] result = inline "+#{content}+", content - + result end def _reduce_16(val, _values, result) content = val[1] result = inline "#{content}", content - + result end @@ -791,13 +791,13 @@ def _reduce_17(val, _values, result) label = val[1] @block_parser.add_label label.reference result = "#{label}" - + result end def _reduce_18(val, _values, result) result = "{#{val[1]}}[#{val[2].join}]" - + result end @@ -805,13 +805,13 @@ def _reduce_19(val, _values, result) scheme, inline = val[1] result = "{#{inline}}[#{scheme}#{inline.reference}]" - + result end def _reduce_20(val, _values, result) result = [nil, inline(val[1])] - + result end @@ -820,25 +820,25 @@ def _reduce_21(val, _values, result) 'rdoc-label:', inline("#{val[0].reference}/#{val[1].reference}") ] - + result end def _reduce_22(val, _values, result) result = ['rdoc-label:', val[0].reference] - + result end def _reduce_23(val, _values, result) result = ['rdoc-label:', "#{val[0].reference}/"] - + result end def _reduce_24(val, _values, result) result = [nil, inline(val[1])] - + result end @@ -847,92 +847,92 @@ def _reduce_25(val, _values, result) 'rdoc-label:', inline("#{val[0].reference}/#{val[1].reference}") ] - + result end def _reduce_26(val, _values, result) result = ['rdoc-label:', val[0]] - + result end def _reduce_27(val, _values, result) ref = val[0].reference result = ['rdoc-label:', inline(ref, "#{ref}/")] - + result end # reduce 28 omitted def _reduce_29(val, _values, result) - result = val[1] + result = val[1] result end def _reduce_30(val, _values, result) - result = val[1] + result = val[1] result end def _reduce_31(val, _values, result) result = inline val[0] - + result end def _reduce_32(val, _values, result) result = inline "\"#{val[1]}\"" - + result end def _reduce_33(val, _values, result) result = inline val[0] - + result end def _reduce_34(val, _values, result) result = inline "\"#{val[1]}\"" - + result end # reduce 35 omitted def _reduce_36(val, _values, result) - result = val[1] + result = val[1] result end def _reduce_37(val, _values, result) - result = inline val[1] + result = inline val[1] result end def _reduce_38(val, _values, result) result = val[0].append val[1] - + result end def _reduce_39(val, _values, result) result = val[0].append val[1] - + result end def _reduce_40(val, _values, result) result = val[0] - + result end def _reduce_41(val, _values, result) result = inline val[0] - + result end @@ -940,25 +940,25 @@ end def _reduce_43(val, _values, result) result = val[0].append val[1] - + result end def _reduce_44(val, _values, result) result = inline val[0] - + result end def _reduce_45(val, _values, result) result = val[0].append val[1] - + result end def _reduce_46(val, _values, result) result = val[0] - + result end @@ -984,24 +984,24 @@ end def _reduce_57(val, _values, result) result = val[0] - + result end def _reduce_58(val, _values, result) result = inline val[0] - + result end def _reduce_59(val, _values, result) result = inline val[0] - + result end def _reduce_60(val, _values, result) - result << val[1] + result << val[1] result end @@ -1009,7 +1009,7 @@ end def _reduce_62(val, _values, result) result << val[1] - + result end @@ -1017,7 +1017,7 @@ end def _reduce_64(val, _values, result) result << val[1] - + result end @@ -1048,7 +1048,7 @@ end # reduce 77 omitted def _reduce_78(val, _values, result) - result << val[1] + result << val[1] result end @@ -1099,13 +1099,13 @@ end def _reduce_101(val, _values, result) index = @block_parser.add_footnote val[1].rdoc result = "{*#{index}}[rdoc-label:foottext-#{index}:footmark-#{index}]" - + result end def _reduce_102(val, _values, result) result = inline "#{val[1]}", val[1] - + result end @@ -1122,7 +1122,7 @@ end # reduce 108 omitted def _reduce_109(val, _values, result) - result << val[1] + result << val[1] result end @@ -1130,24 +1130,24 @@ end def _reduce_111(val, _values, result) result = inline val[0] - + result end # reduce 112 omitted def _reduce_113(val, _values, result) - result = val[1] + result = val[1] result end def _reduce_114(val, _values, result) - result = val[1] + result = val[1] result end def _reduce_115(val, _values, result) - result = val[1] + result = val[1] result end @@ -1192,7 +1192,7 @@ end # reduce 135 omitted def _reduce_136(val, _values, result) - result << val[1] + result << val[1] result end diff --git a/lib/rdoc/rdoc.gemspec b/lib/rdoc/rdoc.gemspec index 1822c27366..4c7cd92549 100644 --- a/lib/rdoc/rdoc.gemspec +++ b/lib/rdoc/rdoc.gemspec @@ -1,11 +1,10 @@ -# -*- encoding: utf-8 -*- -$:.unshift File.expand_path("../lib", __FILE__) -require 'rdoc' +require_relative "lib/rdoc" Gem::Specification.new do |s| s.name = "rdoc" s.version = RDoc::VERSION - + s.date = "2017-09-12" + s.authors = [ "Eric Hodel", "Dave Thomas", @@ -28,7 +27,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat s.executables = ["rdoc", "ri"] s.require_paths = ["lib"] # for ruby core repository. It was generated by `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/gauntlet_rdoc.rb", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml","lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_lex.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"] + s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "appveyor.yml", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ripper_state_lex.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"] # files from .gitignore s.files << "lib/rdoc/rd/block_parser.rb" << "lib/rdoc/rd/inline_parser.rb" << "lib/rdoc/markdown.rb" @@ -54,4 +53,5 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat s.add_development_dependency("racc", "> 1.4.10") s.add_development_dependency("kpeg") s.add_development_dependency("minitest", "~> 4") + s.add_development_dependency("json") end diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb deleted file mode 100644 index e76fdf0414..0000000000 --- a/lib/rdoc/ruby_lex.rb +++ /dev/null @@ -1,1521 +0,0 @@ -# coding: US-ASCII -# frozen_string_literal: false - -#-- -# irb/ruby-lex.rb - ruby lexcal analyzer -# $Release Version: 0.9.5$ -# $Revision: 17979 $ -# $Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $ -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# -#++ - -require "e2mmap" -require "irb/slex" -require "stringio" - -## -# Ruby lexer adapted from irb. -# -# The internals are not documented because they are scary. - -class RDoc::RubyLex - - ## - # Raised upon invalid input - - class Error < RDoc::Error - end - - # :stopdoc: - - extend Exception2MessageMapper - - def_exception(:AlreadyDefinedToken, "Already defined token(%s)") - def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')") - def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')") - def_exception(:TkReading2TokenDuplicateError, - "key duplicate(token_n='%s', key='%s')") - def_exception(:SyntaxError, "%s") - - def_exception(:TerminateLineInput, "Terminate Line Input") - - include RDoc::RubyToken - include IRB - - attr_accessor :continue - attr_accessor :lex_state - attr_accessor :first_in_method_statement - attr_reader :reader - - class << self - attr_accessor :debug_level - end - - def self.debug? - @debug_level > 0 - end - - self.debug_level = 0 - - # :startdoc: - - ## - # Returns an Array of +ruby+ tokens. See ::new for a description of - # +options+. - - def self.tokenize ruby, options - tokens = [] - - scanner = RDoc::RubyLex.new ruby, options - scanner.exception_on_syntax_error = true - - while token = scanner.token do - tokens << token - end - - tokens - end - - ## - # Creates a new lexer for +content+. +options+ is an RDoc::Options, only - # +tab_width is used. - - def initialize(content, options) - lex_init - - if /\t/ =~ content then - tab_width = options.tab_width - content = content.split(/\n/).map do |line| - 1 while line.gsub!(/\t+/) { - ' ' * (tab_width*$&.length - $`.length % tab_width) - } && $~ - line - end.join("\n") - end - - content << "\n" unless content[-1, 1] == "\n" - - set_input StringIO.new content - - @base_char_no = 0 - @char_no = 0 - @exp_line_no = @line_no = 1 - @here_readed = [] - @readed = [] - @current_readed = @readed - @rests = [] - @seek = 0 - - @heredoc_queue = [] - - @indent = 0 - @indent_stack = [] - @lex_state = :EXPR_BEG - @space_seen = false - @escaped_nl = false - @first_in_method_statement = false - @after_question = false - - @continue = false - @line = "" - - @skip_space = false - @readed_auto_clean_up = false - @exception_on_syntax_error = true - - @prompt = nil - @prev_seek = nil - @ltype = nil - end - - # :stopdoc: - - def inspect # :nodoc: - "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [ - self.class, object_id, - @io.pos, @lex_state, @space_seen, - ] - end - - attr_accessor :skip_space - attr_accessor :readed_auto_clean_up - attr_accessor :exception_on_syntax_error - - attr_reader :seek - attr_reader :char_no - attr_reader :line_no - attr_reader :indent - - # io functions - def set_input(io, p = nil, &block) - @io = io - if p.respond_to?(:call) - @input = p - elsif block_given? - @input = block - else - @input = Proc.new{@io.gets} - end - end - - def get_readed - if idx = @readed.rindex("\n") - @base_char_no = @readed.size - (idx + 1) - else - @base_char_no += @readed.size - end - - readed = @readed.join("") - @readed.clear - readed - end - - def getc - while @rests.empty? - # return nil unless buf_input - @rests.push nil unless buf_input - end - c = @rests.shift - @current_readed.push c - @seek += 1 - if c == "\n".freeze - @line_no += 1 - @char_no = 0 - else - @char_no += 1 - end - - c - end - - def gets - l = "" - while c = getc - l.concat(c) - break if c == "\n" - end - return nil if l == "" and c.nil? - l - end - - def eof? - @io.eof? - end - - def getc_of_rests - if @rests.empty? - nil - else - getc - end - end - - def ungetc(c = nil) - if @here_readed.empty? - c2 = @readed.pop - else - c2 = @here_readed.pop - end - c = c2 unless c - @rests.unshift c #c = - @seek -= 1 - if c == "\n" - @line_no -= 1 - if idx = @readed.rindex("\n") - @char_no = idx + 1 - else - @char_no = @base_char_no + @readed.size - end - else - @char_no -= 1 - end - end - - def peek_equal?(str) - chrs = str.split(//) - until @rests.size >= chrs.size - return false unless buf_input - end - @rests[0, chrs.size] == chrs - end - - def peek_match?(regexp) - while @rests.empty? - return false unless buf_input - end - regexp =~ @rests.join("") - end - - def peek(i = 0) - while @rests.size <= i - return nil unless buf_input - end - @rests[i] - end - - def buf_input - prompt - line = @input.call - return nil unless line - @rests.concat line.split(//) - true - end - private :buf_input - - def set_prompt(p = nil, &block) - p = block if block_given? - if p.respond_to?(:call) - @prompt = p - else - @prompt = Proc.new{print p} - end - end - - def prompt - if @prompt - @prompt.call(@ltype, @indent, @continue, @line_no) - end - end - - def initialize_input - @ltype = nil - @quoted = nil - @indent = 0 - @indent_stack = [] - @lex_state = :EXPR_BEG - @space_seen = false - @current_readed = @readed - - @continue = false - prompt - - @line = "" - @exp_line_no = @line_no - end - - def each_top_level_statement - initialize_input - catch(:TERM_INPUT) do - loop do - begin - @continue = false - prompt - unless l = lex - throw :TERM_INPUT if @line == '' - else - #p l - @line.concat l - if @ltype or @continue or @indent > 0 - next - end - end - if @line != "\n" - yield @line, @exp_line_no - end - break unless l - @line = '' - @exp_line_no = @line_no - - @indent = 0 - @indent_stack = [] - prompt - rescue TerminateLineInput - initialize_input - prompt - get_readed - end - end - end - end - - def lex - until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) && - !@continue or - tk.nil?) - #p tk - #p @lex_state - #p self - end - line = get_readed - # print self.inspect - if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil? - nil - else - line - end - end - - def token - # require "tracer" - # Tracer.on - @prev_seek = @seek - @prev_line_no = @line_no - @prev_char_no = @char_no - begin - begin - tk = @OP.match(self) - @space_seen = tk.kind_of?(TkSPACE) - @first_in_method_statement = false if !@space_seen && @first_in_method_statement - rescue SyntaxError => e - raise Error, "syntax error: #{e.message}" if - @exception_on_syntax_error - - tk = TkError.new(@seek, @line_no, @char_no) - end - end while @skip_space and tk.kind_of?(TkSPACE) - - if @readed_auto_clean_up - get_readed - end - - if TkSYMBEG === tk then - tk1 = token - set_token_position tk.seek, tk.line_no, tk.char_no - - case tk1 - when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then - if tk1.respond_to?(:name) then - tk = Token(TkSYMBOL, ":" + tk1.name) - else - tk = Token(TkSYMBOL, ":" + tk1.text) - end - else - tk = tk1 - end - elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then - tk1 = token - set_token_position tk.seek, tk.line_no, tk.char_no - tk = Token(tk1.class, tk.text + tk1.text) - end - @after_question = false if @after_question and !(TkQUESTION === tk) - - # Tracer.off - tk - end - - ENINDENT_CLAUSE = [ - "case", "class", "def", "do", "for", "if", - "module", "unless", "until", "while", "begin" #, "when" - ] - - DEINDENT_CLAUSE = ["end" #, "when" - ] - - PERCENT_LTYPE = { - "q" => "\'", - "Q" => "\"", - "x" => "\`", - "r" => "/", - "w" => "]", - "W" => "]", - "s" => ":", - "i" => "]", - "I" => "]" - } - - PERCENT_PAREN = { - "{" => "}", - "[" => "]", - "<" => ">", - "(" => ")" - } - - PERCENT_PAREN_REV = PERCENT_PAREN.invert - - Ltype2Token = { - "\'" => TkSTRING, - "\"" => TkSTRING, - "\`" => TkXSTRING, - "/" => TkREGEXP, - "]" => TkDSTRING, - ":" => TkSYMBOL - } - DLtype2Token = { - "\"" => TkDSTRING, - "\`" => TkDXSTRING, - "/" => TkDREGEXP, - } - - def lex_init() - @OP = IRB::SLex.new - @OP.def_rules("\0", "\004", "\032") do |op, io| - Token(TkEND_OF_SCRIPT, '') - end - - @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io| - @space_seen = true - str = op - while (ch = getc) =~ /[ \t\f\r\13]/ do - str << ch - end - ungetc - Token TkSPACE, str - end - - @OP.def_rule("#") do |op, io| - identify_comment - end - - @OP.def_rule("=begin", - proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do - |op, io| - @ltype = "=" - res = op - until (ch = getc) == "\n" do - res << ch - end - res << ch - - until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do - (ch = getc) - res << ch - end - - res << gets # consume =end - - @ltype = nil - Token(TkRD_COMMENT, res) - end - - @OP.def_rule("\n") do |op, io| - print "\\n\n" if RDoc::RubyLex.debug? - unless @heredoc_queue.empty? - info = @heredoc_queue[0] - if !info[:started] # "\n" - info[:started] = true - ungetc "\n" - elsif info[:heredoc_end].nil? # heredoc body - tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent]) - info[:heredoc_end] = heredoc_end - ungetc "\n" - else # heredoc end - @heredoc_queue.shift - @lex_state = :EXPR_BEG - tk = Token(TkHEREDOCEND, info[:heredoc_end]) - if !@heredoc_queue.empty? - @heredoc_queue[0][:started] = true - ungetc "\n" - end - end - end - unless tk - case @lex_state - when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT - @continue = true - else - @continue = false - @lex_state = :EXPR_BEG unless @escaped_nl - until (@indent_stack.empty? || - [TkLPAREN, TkLBRACK, TkLBRACE, - TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last)) - @indent_stack.pop - end - end - @current_readed = @readed - @here_readed.clear - tk = Token(TkNL) - end - @escaped_nl = false - tk - end - - @OP.def_rules("=") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - @lex_state = :EXPR_ARG - else - @lex_state = :EXPR_BEG - end - Token(op) - end - - @OP.def_rules("*", "**", - "==", "===", - "=~", "<=>", - "<", "<=", - ">", ">=", ">>", "=>") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - tk - end - - @OP.def_rules("->") do - |op, io| - @lex_state = :EXPR_ENDFN - Token(op) - end - - @OP.def_rules("!", "!=", "!~") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - @lex_state = :EXPR_ARG - Token(TkId, op) - else - @lex_state = :EXPR_BEG - Token(op) - end - end - - @OP.def_rules("<<") do - |op, io| - tk = nil - if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS && - (@lex_state != :EXPR_ARG || @space_seen) - c = peek(0) - if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~") - tk = identify_here_document(op) - end - end - unless tk - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - end - tk - end - - @OP.def_rules("'", '"') do - |op, io| - identify_string(op) - end - - @OP.def_rules("`") do - |op, io| - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - @lex_state = :EXPR_ARG - Token(TkId, op) - else - identify_string(op) - end - end - - @OP.def_rules('?') do - |op, io| - if @lex_state == :EXPR_END - @lex_state = :EXPR_BEG - @after_question = true - Token(TkQUESTION) - else - ch = getc - if @lex_state == :EXPR_ARG && ch =~ /\s/ - ungetc - @lex_state = :EXPR_BEG; - Token(TkQUESTION) - else - @lex_state = :EXPR_END - ch << getc if "\\" == ch - Token(TkCHAR, "?#{ch}") - end - end - end - - @OP.def_rules("&&", "||") do - |op, io| - @lex_state = :EXPR_BEG - Token(op) - end - - @OP.def_rules("&", "|") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - tk - end - - @OP.def_rules("+=", "-=", "*=", "**=", - "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do - |op, io| - @lex_state = :EXPR_BEG - op =~ /^(.*)=$/ - Token(TkOPASGN, $1) - end - - @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rules("+", "-") do - |op, io| - catch(:RET) do - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - elsif @lex_state == :EXPR_ARG - if @space_seen and peek(0) =~ /[0-9]/ - throw :RET, identify_number(op) - else - @lex_state = :EXPR_BEG - end - elsif @lex_state != :EXPR_END and peek(0) =~ /[0-9]/ - throw :RET, identify_number(op) - else - @lex_state = :EXPR_BEG - end - tk = Token(op) unless tk - tk - end - end - - @OP.def_rules(".", "&.") do - |op, io| - @lex_state = :EXPR_BEG - if peek(0) =~ /[0-9]/ - ungetc - identify_number - else - # for "obj.if" or "obj&.if" etc. - @lex_state = :EXPR_DOT - Token(op) - end - end - - @OP.def_rules("..", "...") do - |op, io| - @lex_state = :EXPR_BEG - Token(op) - end - - lex_int2 - end - - def lex_int2 - @OP.def_rules("]", "}", ")") do - |op, io| - @lex_state = :EXPR_END - @indent -= 1 - @indent_stack.pop - Token(op) - end - - @OP.def_rule(":") do - |op, io| - if @lex_state == :EXPR_END || peek(0) =~ /\s/ - @lex_state = :EXPR_BEG - Token(TkCOLON) - else - @lex_state = :EXPR_FNAME; - Token(TkSYMBEG) - end - end - - @OP.def_rule("::") do - |op, io| - # p @lex_state.id2name, @space_seen - if @lex_state == :EXPR_BEG or @lex_state == :EXPR_ARG && @space_seen - @lex_state = :EXPR_BEG - Token(TkCOLON3) - else - @lex_state = :EXPR_DOT - Token(TkCOLON2) - end - end - - @OP.def_rule("/") do - |op, io| - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - @lex_state = :EXPR_ARG - Token(TkId, op) - elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement - identify_string(op) - elsif peek(0) == '=' - getc - @lex_state = :EXPR_BEG - Token(TkOPASGN, "/") #/) - elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/ - identify_string(op) - else - @lex_state = :EXPR_BEG - Token("/") #/) - end - end - - @OP.def_rules("^") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - tk - end - - # @OP.def_rules("^=") do - # @lex_state = :EXPR_BEG - # Token(OP_ASGN, :^) - # end - - @OP.def_rules(",") do - |op, io| - @lex_state = :EXPR_BEG - Token(op) - end - - @OP.def_rules(";") do - |op, io| - @lex_state = :EXPR_BEG - until (@indent_stack.empty? || - [TkLPAREN, TkLBRACK, TkLBRACE, - TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last)) - @indent_stack.pop - end - Token(op) - end - - @OP.def_rule("~") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - @lex_state = :EXPR_ARG - Token(TkId, op) - else - @lex_state = :EXPR_BEG - Token(op) - end - end - - @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_BEG - Token("~") - end - - @OP.def_rule("(") do - |op, io| - @indent += 1 - if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID - @lex_state = :EXPR_BEG - tk_c = TkfLPAREN - else - @lex_state = :EXPR_BEG - tk_c = TkLPAREN - end - @indent_stack.push tk_c - Token tk_c - end - - @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rule("[") do - |op, io| - text = nil - @indent += 1 - if @lex_state == :EXPR_FNAME - tk_c = TkfLBRACK - else - if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID - tk_c = TkLBRACK - elsif @lex_state == :EXPR_ARG && @space_seen - tk_c = TkLBRACK - elsif @lex_state == :EXPR_DOT - if peek(0) == "]" - tk_c = TkIDENTIFIER - getc - if peek(0) == "=" - text = "[]=" - else - text = "[]" - end - else - tk_c = TkOp - end - else - tk_c = TkfLBRACK - end - @lex_state = :EXPR_BEG - end - @indent_stack.push tk_c - Token(tk_c, text) - end - - @OP.def_rule("{") do - |op, io| - @indent += 1 - if @lex_state != :EXPR_END && @lex_state != :EXPR_ARG - tk_c = TkLBRACE - else - tk_c = TkfLBRACE - end - @lex_state = :EXPR_BEG - @indent_stack.push tk_c - Token(tk_c) - end - - @OP.def_rule('\\') do - |op, io| - if peek(0) == "\n" - @space_seen = true - @continue = true - @escaped_nl = true - end - Token("\\") - end - - @OP.def_rule('%') do - |op, io| - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - @lex_state = :EXPR_ARG - Token(TkId, op) - elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID - identify_quotation - elsif peek(0) == '=' - getc - @lex_state = :EXPR_BEG - Token(TkOPASGN, '%') - elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/ - identify_quotation - else - @lex_state = :EXPR_BEG - Token("%") #)) - end - end - - @OP.def_rule('$') do - |op, io| - identify_gvar - end - - @OP.def_rule('@') do - |op, io| - if peek(0) =~ /[\w@]/ - ungetc - identify_identifier - else - Token("@") - end - end - - # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do - # |op, io| - # @indent += 1 - # @lex_state = :EXPR_FNAME - # # @lex_state = :EXPR_END - # # until @rests[0] == "\n" or @rests[0] == ";" - # # rests.shift - # # end - # end - - @OP.def_rule("_") do - if peek_match?(/_END__/) and @lex_state == :EXPR_BEG then - 6.times { getc } - Token(TkEND_OF_SCRIPT, '__END__') - else - ungetc - identify_identifier - end - end - - @OP.def_rule("") do - |op, io| - printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug? - if peek(0) =~ /[0-9]/ - t = identify_number - else - t = identify_identifier - end - printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug? - t - end - - p @OP if RDoc::RubyLex.debug? - end - - def identify_gvar - @lex_state = :EXPR_END - - case ch = getc - when /[~_*$?!@\/\\;,=:<>".]/ #" - Token(TkGVAR, "$" + ch) - when "-" - Token(TkGVAR, "$-" + getc) - when "&", "`", "'", "+" - Token(TkBACK_REF, "$"+ch) - when /[1-9]/ - ref = ch - while (ch = getc) =~ /[0-9]/ do ref << ch end - ungetc - Token(TkNTH_REF, "$#{ref}") - when /\w/ - ungetc - ungetc - identify_identifier - else - ungetc - Token("$") - end - end - - IDENT_RE = eval '/[\w\u{0080}-\u{FFFFF}]/u' - - def identify_identifier - token = "" - if peek(0) =~ /[$@]/ - token.concat(c = getc) - if c == "@" and peek(0) == "@" - token.concat getc - end - end - - while (ch = getc) =~ IDENT_RE do - print " :#{ch}: " if RDoc::RubyLex.debug? - token.concat ch - end - - ungetc - - if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/ - token.concat getc - end - - # almost fix token - - case token - when /^\$/ - return Token(TkGVAR, token) - when /^\@\@/ - @lex_state = :EXPR_END - # p Token(TkCVAR, token) - return Token(TkCVAR, token) - when /^\@/ - @lex_state = :EXPR_END - return Token(TkIVAR, token) - end - - if @lex_state != :EXPR_DOT - print token, "\n" if RDoc::RubyLex.debug? - - token_c, *trans = TkReading2Token[token] - if token_c - # reserved word? - - if (@lex_state != :EXPR_BEG && - @lex_state != :EXPR_FNAME && - trans[1]) - # modifiers - token_c = TkSymbol2Token[trans[1]] - @lex_state = trans[0] - else - if @lex_state != :EXPR_FNAME - if ENINDENT_CLAUSE.include?(token) - valid = peek(0) != ':' - - # check for ``class = val'' etc. - case token - when "class" - valid = false unless peek_match?(/^\s*(<<|\w|::)/) - when "def" - valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/) - when "do" - valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/) - when *ENINDENT_CLAUSE - valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/) - else - # no nothing - end if valid - - if valid - if token == "do" - if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last) - @indent += 1 - @indent_stack.push token_c - end - else - @indent += 1 - @indent_stack.push token_c - end - else - token_c = TkIDENTIFIER - end - - elsif DEINDENT_CLAUSE.include?(token) - @indent -= 1 - @indent_stack.pop - end - @lex_state = trans[0] - else - @lex_state = :EXPR_END - end - end - if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/) - token.concat getc - token_c = TkSYMBOL - end - return Token(token_c, token) - end - end - - if @lex_state == :EXPR_FNAME - @lex_state = :EXPR_END - if peek(0) == '=' and peek(1) != '>' - token.concat getc - end - elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT || - @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID - @lex_state = :EXPR_ARG - else - @lex_state = :EXPR_END - end - - if token[0, 1] =~ /[A-Z]/ - if token[-1] =~ /[!?]/ - token_c = TkIDENTIFIER - else - token_c = TkCONSTANT - end - elsif token[token.size - 1, 1] =~ /[!?]/ - token_c = TkFID - else - token_c = TkIDENTIFIER - end - if peek(0) == ':' and !peek_match?(/^::/) - token.concat getc - return Token(TkSYMBOL, token) - else - return Token(token_c, token) - end - end - - def identify_here_document(op) - ch = getc - start_token = op - # if lt = PERCENT_LTYPE[ch] - if ch == "-" or ch == "~" - start_token.concat ch - ch = getc - indent = true - end - if /['"`]/ =~ ch - start_token.concat ch - user_quote = lt = ch - quoted = "" - while (c = getc) && c != lt - quoted.concat c - end - start_token.concat quoted - start_token.concat lt - else - user_quote = nil - lt = '"' - quoted = ch.dup - while (c = getc) && c =~ /\w/ - quoted.concat c - end - start_token.concat quoted - ungetc - end - - @heredoc_queue << { - quoted: quoted, - lt: lt, - indent: indent, - started: false - } - @lex_state = :EXPR_END - Token(RDoc::RubyLex::TkHEREDOCBEG, start_token) - end - - def identify_here_document_body(quoted, lt, indent) - ltback, @ltype = @ltype, lt - - doc = "" - heredoc_end = nil - while l = gets - l = l.sub(/(:?\r)?\n\z/, "\n") - if (indent ? l.strip : l.chomp) == quoted - heredoc_end = l - break - end - doc << l - end - raise Error, "Missing terminating #{quoted} for string" unless heredoc_end - - @ltype = ltback - @lex_state = :EXPR_BEG - [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end] - end - - def identify_quotation - type = ch = getc - if lt = PERCENT_LTYPE[type] - ch = getc - elsif type =~ /\W/ - type = nil - lt = "\"" - else - return Token(TkMOD, '%') - end - # if ch !~ /\W/ - # ungetc - # next - # end - #@ltype = lt - @quoted = ch unless @quoted = PERCENT_PAREN[ch] - identify_string(lt, @quoted, type) - end - - def identify_number(op = "") - @lex_state = :EXPR_END - - num = op - - if peek(0) == "0" && peek(1) !~ /[.eEri]/ - num << getc - - case peek(0) - when /[xX]/ - ch = getc - match = /[0-9a-fA-F_]/ - when /[bB]/ - ch = getc - match = /[01_]/ - when /[oO]/ - ch = getc - match = /[0-7_]/ - when /[dD]/ - ch = getc - match = /[0-9_]/ - when /[0-7]/ - match = /[0-7_]/ - when /[89]/ - raise Error, "Illegal octal digit" - else - return Token(TkINTEGER, num) - end - - num << ch if ch - - len0 = true - non_digit = false - while ch = getc - num << ch - if match =~ ch - if ch == "_" - if non_digit - raise Error, "trailing `#{ch}' in number" - else - non_digit = ch - end - else - non_digit = false - len0 = false - end - else - ungetc - num[-1, 1] = '' - if len0 - raise Error, "numeric literal without digits" - end - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - break - end - end - return Token(TkINTEGER, num) - end - - type = TkINTEGER - allow_point = true - allow_e = true - allow_ri = true - non_digit = false - while ch = getc - num << ch - case ch - when /[0-9]/ - non_digit = false - when "_" - non_digit = ch - when allow_point && "." - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - type = TkFLOAT - if peek(0) !~ /[0-9]/ - type = TkINTEGER - ungetc - num[-1, 1] = '' - break - end - allow_point = false - when allow_e && "e", allow_e && "E" - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - type = TkFLOAT - if peek(0) =~ /[+-]/ - num << getc - end - allow_e = false - allow_ri = false - allow_point = false - non_digit = ch - when allow_ri && "r" - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - type = TkRATIONAL - if peek(0) == 'i' - type = TkIMAGINARY - num << getc - end - break - when allow_ri && "i" - if non_digit && non_digit != "r" - raise Error, "trailing `#{non_digit}' in number" - end - type = TkIMAGINARY - break - else - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - ungetc - num[-1, 1] = '' - break - end - end - - Token(type, num) - end - - def identify_string(ltype, quoted = ltype, type = nil) - close = PERCENT_PAREN.values.include?(quoted) - @ltype = ltype - @quoted = quoted - - str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then - ltype.dup - else - "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}" - end - - subtype = nil - begin - nest = 0 - - while ch = getc - str << ch - - if @quoted == ch and nest <= 0 - break - elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#" - ch = getc - if ch == "{" then - subtype = true - str << ch << skip_inner_expression - next - else - ungetc - end - elsif ch == '\\' - case @ltype - when "'" then - case ch = getc - when "'", '\\' then - str << ch - else - str << ch - end - else - str << read_escape - end - end - - if close then - if PERCENT_PAREN[ch] == @quoted - nest += 1 - elsif ch == @quoted - nest -= 1 - end - end - end - - if @ltype == "/" - while peek(0) =~ /i|m|x|o|e|s|u|n/ - str << getc - end - end - - if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question - str.concat getc - return Token(TkSYMBOL, str) - elsif subtype - Token(DLtype2Token[ltype], str) - else - Token(Ltype2Token[ltype], str) - end - ensure - @ltype = nil - @quoted = nil - @lex_state = :EXPR_END - end - end - - def skip_inner_expression - res = "" - nest = 0 - while ch = getc - res << ch - if ch == '}' - break if nest.zero? - nest -= 1 - elsif ch == '{' - nest += 1 - end - end - res - end - - def identify_comment - @ltype = "#" - - comment = '#' - - while ch = getc - # if ch == "\\" #" - # read_escape - # end - if ch == "\n" - @ltype = nil - ungetc - break - end - - comment << ch - end - - return Token(TkCOMMENT, comment) - end - - def read_escape - escape = '' - ch = getc - - case ch - when "\n", "\r", "\f" - escape << ch - when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #" - escape << ch - when /[0-7]/ - ungetc ch - 3.times do - ch = getc - case ch - when /[0-7]/ - escape << ch - when nil - break - else - ungetc - break - end - end - - when "x" - escape << ch - - 2.times do - ch = getc - case ch - when /[0-9a-fA-F]/ - escape << ch - when nil - break - else - ungetc - break - end - end - - when "M" - escape << ch - - ch = getc - if ch != '-' - ungetc - else - escape << ch - - ch = getc - if ch == "\\" #" - ungetc - escape << read_escape - else - escape << ch - end - end - - when "C", "c" #, "^" - escape << ch - - if ch == "C" - ch = getc - - if ch == "-" - escape << ch - ch = getc - escape << ch - - escape << read_escape if ch == "\\" - else - ungetc - end - elsif (ch = getc) == "\\" #" - escape << ch << read_escape - end - else - escape << ch - - # other characters - end - - escape - end - - # :startdoc: - -end - -#RDoc::RubyLex.debug_level = 1 diff --git a/lib/rdoc/stats/normal.rb b/lib/rdoc/stats/normal.rb index ba00b6cbdf..8ad0e453c6 100644 --- a/lib/rdoc/stats/normal.rb +++ b/lib/rdoc/stats/normal.rb @@ -1,5 +1,5 @@ # frozen_string_literal: false -require 'io/console/size' +require 'io/console' ## # Stats printer that prints just the files being documented with a progress @@ -23,7 +23,7 @@ class RDoc::Stats::Normal < RDoc::Stats::Quiet # Print a progress bar, but make sure it fits on a single line. Filename # will be truncated if necessary. - terminal_width = IO.console_size[1].to_i.nonzero? || 80 + terminal_width = IO.console.winsize[1].to_i.nonzero? || 80 max_filename_size = terminal_width - progress_bar.size if filename.size > max_filename_size then diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 0cfa2f1384..df5232f7ea 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -30,28 +30,41 @@ module RDoc::TokenStream token_stream.map do |t| next unless t - style = case t - when RDoc::RubyToken::TkCONSTANT then 'ruby-constant' - when RDoc::RubyToken::TkKW then 'ruby-keyword' - when RDoc::RubyToken::TkIVAR then 'ruby-ivar' - when RDoc::RubyToken::TkOp then 'ruby-operator' - when RDoc::RubyToken::TkId then 'ruby-identifier' - when RDoc::RubyToken::TkREGEXP then 'ruby-regexp' - when RDoc::RubyToken::TkDREGEXP then 'ruby-regexp' - when RDoc::RubyToken::TkNode then 'ruby-node' - when RDoc::RubyToken::TkCOMMENT then 'ruby-comment' - when RDoc::RubyToken::TkXSTRING then 'ruby-string' - when RDoc::RubyToken::TkSTRING then 'ruby-string' - when RDoc::RubyToken::TkVal then 'ruby-value' + style = case t[:kind] + when :on_const then 'ruby-constant' + when :on_kw then 'ruby-keyword' + when :on_ivar then 'ruby-ivar' + when :on_cvar then 'ruby-identifier' + when :on_gvar then 'ruby-identifier' + when '=' != t[:text] && :on_op then + if RDoc::RipperStateLex::EXPR_ARG == t[:state] then + 'ruby-identifier' + else + 'ruby-operator' + end + when :on_tlambda then 'ruby-operator' + when :on_ident then 'ruby-identifier' + when :on_label then 'ruby-value' + when :on_backref, :on_dstring + then 'ruby-node' + when :on_comment then 'ruby-comment' + when :on_embdoc then 'ruby-comment' + when :on_regexp then 'ruby-regexp' + when :on_tstring then 'ruby-string' + when :on_int, :on_float, + :on_rational, :on_imaginary, + :on_heredoc, + :on_symbol, :on_CHAR then 'ruby-value' + when :on_heredoc_beg, :on_heredoc_end + then 'ruby-identifier' end comment_with_nl = false - case t - when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND - comment_with_nl = true if t.text =~ /\n$/ - text = t.text.rstrip + if :on_comment == t[:kind] or :on_embdoc == t[:kind] or :on_heredoc_end == t[:kind] + comment_with_nl = true if "\n" == t[:text][-1] + text = t[:text].rstrip else - text = t.text + text = t[:text] end text = CGI.escapeHTML text diff --git a/test/rdoc/test_rdoc_any_method.rb b/test/rdoc/test_rdoc_any_method.rb index fdbb62efa6..fdfb0f3995 100644 --- a/test/rdoc/test_rdoc_any_method.rb +++ b/test/rdoc/test_rdoc_any_method.rb @@ -74,7 +74,7 @@ method(a, b) { |c, d| ... } def test_markup_code tokens = [ - RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'), + { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' }, ] @c2_a.collect_tokens diff --git a/test/rdoc/test_rdoc_constant.rb b/test/rdoc/test_rdoc_constant.rb index c43aa7dc2b..a03ab97a7d 100644 --- a/test/rdoc/test_rdoc_constant.rb +++ b/test/rdoc/test_rdoc_constant.rb @@ -86,7 +86,7 @@ class TestRDocConstant < XrefTestCase assert_equal top_level, loaded.file assert_equal 'Klass::CONST', loaded.full_name assert_equal 'CONST', loaded.name - assert_nil loaded.visibility + assert_equal :public, loaded.visibility assert_equal cm, loaded.parent assert_equal section, loaded.section end @@ -114,7 +114,7 @@ class TestRDocConstant < XrefTestCase assert_equal top_level, loaded.file assert_equal 'Klass::CONST', loaded.full_name assert_equal 'CONST', loaded.name - assert_nil loaded.visibility + assert_equal :public, loaded.visibility assert_equal cm, loaded.parent assert_equal section, loaded.section @@ -146,7 +146,7 @@ class TestRDocConstant < XrefTestCase assert_equal top_level, loaded.file assert_equal 'Klass::CONST', loaded.full_name assert_equal 'CONST', loaded.name - assert_nil loaded.visibility + assert_equal :public, loaded.visibility assert_equal cm, loaded.parent assert_equal section, loaded.section diff --git a/test/rdoc/test_rdoc_context.rb b/test/rdoc/test_rdoc_context.rb index 54e365f7f4..ceac14ac90 100644 --- a/test/rdoc/test_rdoc_context.rb +++ b/test/rdoc/test_rdoc_context.rb @@ -719,6 +719,7 @@ class TestRDocContext < XrefTestCase assert_equal [@pub, @prot, @priv], @vis.method_list assert_equal [@apub, @aprot, @apriv], @vis.attributes + assert_equal [@cpub, @cpriv], @vis.constants end def test_remove_invisible_nodoc @@ -728,6 +729,7 @@ class TestRDocContext < XrefTestCase assert_equal [@pub, @prot, @priv], @vis.method_list assert_equal [@apub, @aprot, @apriv], @vis.attributes + assert_equal [@cpub, @cpriv], @vis.constants end def test_remove_invisible_protected @@ -737,6 +739,7 @@ class TestRDocContext < XrefTestCase assert_equal [@pub, @prot], @vis.method_list assert_equal [@apub, @aprot], @vis.attributes + assert_equal [@cpub], @vis.constants end def test_remove_invisible_public @@ -746,6 +749,7 @@ class TestRDocContext < XrefTestCase assert_equal [@pub], @vis.method_list assert_equal [@apub], @vis.attributes + assert_equal [@cpub], @vis.constants end def test_remove_invisible_public_force @@ -755,11 +759,13 @@ class TestRDocContext < XrefTestCase @prot.force_documentation = true @apriv.force_documentation = true @aprot.force_documentation = true + @cpriv.force_documentation = true @vis.remove_invisible :public assert_equal [@pub, @prot, @priv], @vis.method_list assert_equal [@apub, @aprot, @apriv], @vis.attributes + assert_equal [@cpub, @cpriv], @vis.constants end def test_remove_invisible_in_protected @@ -922,6 +928,9 @@ class TestRDocContext < XrefTestCase @aprot = RDoc::Attr.new nil, 'prot', 'RW', nil @apriv = RDoc::Attr.new nil, 'priv', 'RW', nil + @cpub = RDoc::Constant.new 'CONST_PUBLIC', nil, nil + @cpriv = RDoc::Constant.new 'CONST_PRIVATE', nil, nil + @vis = RDoc::NormalClass.new 'Vis' @vis.add_method @pub @vis.add_method @prot @@ -931,11 +940,16 @@ class TestRDocContext < XrefTestCase @vis.add_attribute @aprot @vis.add_attribute @apriv + @vis.add_constant @cpub + @vis.add_constant @cpriv + @prot.visibility = :protected @priv.visibility = :private @aprot.visibility = :protected @apriv.visibility = :private + + @cpriv.visibility = :private end end diff --git a/test/rdoc/test_rdoc_markup_to_html.rb b/test/rdoc/test_rdoc_markup_to_html.rb index fd8a7bb0ca..b5edeb04d2 100644 --- a/test/rdoc/test_rdoc_markup_to_html.rb +++ b/test/rdoc/test_rdoc_markup_to_html.rb @@ -452,6 +452,9 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase end def test_accept_verbatim_nl_after_backslash + # TODO: Remove "skip" after the issue is resolved: https://github.com/jruby/jruby/issues/4787 + # This "skip" is for strange behavior around escaped newline on JRuby + skip if defined? JRUBY_VERSION verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", " this_is_flag_var\n") @to.start_accepting diff --git a/test/rdoc/test_rdoc_parser_c.rb b/test/rdoc/test_rdoc_parser_c.rb index d0732597dc..5341da16e2 100644 --- a/test/rdoc/test_rdoc_parser_c.rb +++ b/test/rdoc/test_rdoc_parser_c.rb @@ -1037,7 +1037,7 @@ Init_Foo(void) { other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "VALUE\nother_function() {\n}", code end @@ -1107,7 +1107,7 @@ Init_Foo(void) { other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "VALUE\nother_function() {\n}", code end @@ -1141,7 +1141,7 @@ Init_Foo(void) { assert_equal '()', other_function.params assert_equal 8, other_function.line - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "VALUE\nrb_other_function() {\n}", code end @@ -1174,7 +1174,7 @@ Init_Foo(void) { assert_equal '()', other_function.params assert_equal 4, other_function.line - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "#define other_function rb_other_function", code end @@ -1314,7 +1314,7 @@ Init_Foo(void) { other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "DLL_LOCAL VALUE\nother_function() {\n}", code end diff --git a/test/rdoc/test_rdoc_parser_ruby.rb b/test/rdoc/test_rdoc_parser_ruby.rb index 55c03d4628..cda407ed51 100644 --- a/test/rdoc/test_rdoc_parser_ruby.rb +++ b/test/rdoc/test_rdoc_parser_ruby.rb @@ -74,7 +74,7 @@ class C; end comment = parser.collect_first_comment - assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment + assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n", @top_level), comment end def test_get_class_or_module @@ -84,7 +84,7 @@ class C; end cont, name_t, given_name = util_parser('A') .get_class_or_module ctxt assert_equal ctxt, cont - assert_equal 'A', name_t.text + assert_equal 'A', name_t[:text] assert_equal 'A', given_name cont, name_t, given_name = util_parser('B::C') .get_class_or_module ctxt @@ -92,16 +92,16 @@ class C; end b = @store.find_module_named('B') assert_equal b, cont assert_equal [@top_level], b.in_files - assert_equal 'C', name_t.text + assert_equal 'C', name_t[:text] assert_equal 'B::C', given_name cont, name_t, given_name = util_parser('D:: E').get_class_or_module ctxt assert_equal @store.find_module_named('D'), cont - assert_equal 'E', name_t.text + assert_equal 'E', name_t[:text] assert_equal 'D::E', given_name - assert_raises NoMethodError do + assert_raises RDoc::Error do util_parser("A::\nB").get_class_or_module ctxt end end @@ -1194,10 +1194,12 @@ EOF assert_equal klass.current_section, foo.section stream = [ - tk(:COMMENT, 0, 1, 1, nil, - "# File #{@top_level.relative_name}, line 1"), - RDoc::Parser::Ruby::NEWLINE_TOKEN, - tk(:SPACE, 0, 1, 1, nil, ''), + { + :line_no => 1, :char_no => 1, :kind => :on_comment, + :text => "# File #{@top_level.relative_name}, line 1" + }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' } ] assert_equal stream, foo.token_stream @@ -1358,6 +1360,33 @@ A::B::C = 1 assert_equal 'comment', c.comment end + def test_parse_constant_with_bracket + util_parser <<-RUBY +class Klass +end + +class Klass2 + CONSTANT = Klass +end + +class Klass3 + CONSTANT_2 = {} + CONSTANT_2[1] = Klass +end + RUBY + + @parser.scan + + klass = @store.find_class_named 'Klass' + klass2 = @store.find_class_named 'Klass2' + klass3 = @store.find_class_named 'Klass3' + constant = klass2.find_module_named 'CONSTANT' + constant2 = klass3.find_module_named 'CONSTANT_2' + assert_equal klass, klass2.constants.first.is_alias_for + refute_equal klass, klass3.constants.first.is_alias_for + assert_nil klass3.find_module_named 'CONSTANT_2' + end + def test_parse_extend_or_include_extend klass = RDoc::NormalClass.new 'C' klass.parent = @top_level @@ -1434,20 +1463,30 @@ A::B::C = 1 assert_equal klass.current_section, foo.section stream = [ - tk(:COMMENT, 0, 1, 1, nil, - "# File #{@top_level.relative_name}, line 1"), - RDoc::Parser::Ruby::NEWLINE_TOKEN, - tk(:SPACE, 0, 1, 1, nil, ''), - tk(:IDENTIFIER, 0, 1, 0, 'add_my_method', 'add_my_method'), - tk(:SPACE, 0, 1, 13, nil, ' '), - tk(:SYMBOL, 0, 1, 14, nil, ':foo'), - tk(:COMMA, 0, 1, 18, nil, ','), - tk(:SPACE, 0, 1, 19, nil, ' '), - tk(:SYMBOL, 0, 1, 20, nil, ':bar'), - tk(:NL, 0, 1, 24, nil, "\n"), + { + :line_no => 1, :char_no => 1, :kind => :on_comment, + :text => "# File #{@top_level.relative_name}, line 1" + }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }, + { :line_no => 1, :char_no => 0, :kind => :on_ident, :text => 'add_my_method' }, + { :line_no => 1, :char_no => 13, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 14, :kind => :on_symbol, :text => ':foo' }, + { :line_no => 1, :char_no => 18, :kind => :on_comma, :text => ',' }, + { :line_no => 1, :char_no => 19, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' }, + { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" } ] + parsed_stream = foo.token_stream.map { |t| + { + :line_no => t[:line_no], + :char_no => t[:char_no], + :kind => t[:kind], + :text => t[:text] + } + } - assert_equal stream, foo.token_stream + assert_equal stream, parsed_stream end def test_parse_meta_method_block @@ -1468,7 +1507,10 @@ end @parser.parse_meta_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment - assert_equal tk(:NL, 0, 3, 3, 3, "\n"), @parser.get_tk + rest = { :line_no => 3, :char_no => 3, :kind => :on_nl, :text => "\n" } + tk = @parser.get_tk + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => tk[:kind], :text => tk[:text] } + assert_equal rest, tk end def test_parse_meta_method_define_method @@ -1631,23 +1673,30 @@ end assert_equal klass.current_section, foo.section stream = [ - tk(:COMMENT, 0, 1, 1, nil, - "# File #{@top_level.relative_name}, line 1"), - RDoc::Parser::Ruby::NEWLINE_TOKEN, - tk(:SPACE, 0, 1, 1, nil, ''), - tk(:DEF, 0, 1, 0, 'def', 'def'), - tk(:SPACE, 3, 1, 3, nil, ' '), - tk(:IDENTIFIER, 4, 1, 4, 'foo', 'foo'), - tk(:LPAREN, 7, 1, 7, nil, '('), - tk(:RPAREN, 8, 1, 8, nil, ')'), - tk(:SPACE, 9, 1, 9, nil, ' '), - tk(:COLON, 10, 1, 10, nil, ':'), - tk(:IDENTIFIER, 11, 1, 11, 'bar', 'bar'), - tk(:SPACE, 14, 1, 14, nil, ' '), - tk(:END, 15, 1, 15, 'end', 'end'), + { + :line_no => 1, :char_no => 1, :kind => :on_comment, + :text => "# File #{@top_level.relative_name}, line 1" }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }, + { :line_no => 1, :char_no => 0, :kind => :on_kw, :text => 'def' }, + { :line_no => 1, :char_no => 3, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 4, :kind => :on_ident, :text => 'foo' }, + { :line_no => 1, :char_no => 7, :kind => :on_lparen, :text => '(' }, + { :line_no => 1, :char_no => 8, :kind => :on_rparen, :text => ')' }, + { :line_no => 1, :char_no => 9, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 10, :kind => :on_symbol, :text => ':bar' }, + { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' } ] - - assert_equal stream, foo.token_stream + parsed_stream = foo.token_stream.map { |t| + { + :line_no => t[:line_no], + :char_no => t[:char_no], + :kind => t[:kind], + :text => t[:text] + } + } + assert_equal stream, parsed_stream end def test_parse_redefinable_methods @@ -1664,8 +1713,8 @@ end end klass.method_list.each do |method| - assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5] - assert_includes redefinable_ops, method.token_stream[5].text + assert_equal :on_ident, method.token_stream[5][:kind] + assert_includes redefinable_ops, method.token_stream[5][:text] end end @@ -1909,6 +1958,20 @@ end assert_equal '(arg1, arg2, arg3)', foo.params end + def test_parse_method_parameters_with_paren_comment_continue + klass = RDoc::NormalClass.new 'Foo' + klass.parent = @top_level + + util_parser "def foo(arg1, arg2, # some useful comment\narg3)\nend" + + tk = @parser.get_tk + + @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, @comment + + foo = klass.method_list.first + assert_equal '(arg1, arg2, arg3)', foo.params + end + def test_parse_method_star klass = RDoc::NormalClass.new 'Foo' klass.parent = @top_level @@ -2091,22 +2154,34 @@ end assert_equal 2, x.method_list.length a = x.method_list.first - expected = [ - tk(:COMMENT, 0, 2, 1, nil, "# File #{@filename}, line 2"), - tk(:NL, 0, 0, 0, nil, "\n"), - tk(:SPACE, 0, 1, 1, nil, ''), - tk(:DEF, 8, 2, 0, 'def', 'def'), - tk(:SPACE, 11, 2, 3, nil, ' '), - tk(:IDENTIFIER, 12, 2, 4, 'a', 'a'), - tk(:NL, 13, 2, 5, nil, "\n"), - tk(:REGEXP, 14, 3, 0, nil, '%r{#}'), - tk(:NL, 19, 3, 5, nil, "\n"), - tk(:DREGEXP, 20, 4, 0, nil, '%r{#{}}'), - tk(:NL, 27, 4, 7, nil, "\n"), - tk(:END, 28, 5, 0, 'end', 'end'), - ] - assert_equal expected, a.token_stream + expected = [ + { + :line_no => 2, :char_no => 1, :kind => :on_comment, + :text => "# File #{@filename}, line 2" + }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }, + { :line_no => 2, :char_no => 0, :kind => :on_kw, :text => 'def' }, + { :line_no => 2, :char_no => 3, :kind => :on_sp, :text => ' ' }, + { :line_no => 2, :char_no => 4, :kind => :on_ident, :text => 'a' }, + { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" }, + { :line_no => 3, :char_no => 0, :kind => :on_regexp, :text => '%r{#}' }, + { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" }, + { :line_no => 4, :char_no => 0, :kind => :on_regexp, :text => '%r{#{}}' }, + { :line_no => 4, :char_no => 7, :kind => :on_nl, :text => "\n" }, + { :line_no => 5, :char_no => 0, :kind => :on_kw, :text => 'end' } + ] + parsed_stream = a.token_stream.map { |tk| + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => tk[:kind], + :text => tk[:text] + } + } + + assert_equal expected, parsed_stream end def test_parse_statements_encoding @@ -2286,6 +2361,9 @@ class Foo SIXTH_CONSTANT = #{sixth_constant} SEVENTH_CONSTANT = proc { |i| begin i end } + + EIGHTH_CONSTANT = "a" \\ + "b" end EOF @@ -2331,6 +2409,11 @@ EOF assert_equal 'SEVENTH_CONSTANT', constant.name assert_equal "proc { |i| begin i end }", constant.value assert_equal @top_level, constant.file + + constant = constants[7] + assert_equal 'EIGHTH_CONSTANT', constant.name + assert_equal "\"a\" \\\n\"b\"", constant.value + assert_equal @top_level, constant.file end def test_parse_statements_identifier_attr @@ -2504,7 +2587,7 @@ EXPTECTED util_parser <def blah() - <<~EOM if true + <<-EOM if true EOM end EXPTECTED @@ -2525,7 +2608,7 @@ EXPTECTED blah = foo.method_list.first markup_code = blah.markup_code.sub(/^.*\n/, '') - assert_equal markup_code, expected + assert_equal expected, markup_code end def test_parse_statements_method_oneliner_with_regexp @@ -2770,17 +2853,21 @@ RUBY end def test_parse_symbol_in_arg - util_parser ':blah "blah" "#{blah}" blah' + util_parser '[:blah, "blah", "#{blah}", blah]' + @parser.get_tk # skip '[' assert_equal 'blah', @parser.parse_symbol_in_arg + @parser.get_tk # skip ',' @parser.skip_tkspace assert_equal 'blah', @parser.parse_symbol_in_arg + @parser.get_tk # skip ',' @parser.skip_tkspace assert_equal nil, @parser.parse_symbol_in_arg + @parser.get_tk # skip ',' @parser.skip_tkspace @@ -2888,7 +2975,7 @@ end assert_equal 'category', directive assert_equal 'test', value - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_allow @@ -2898,7 +2985,7 @@ end assert_nil directive - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_empty @@ -2908,7 +2995,7 @@ end assert_nil directive - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_no_comment @@ -2918,18 +3005,18 @@ end assert_nil directive - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_one_liner - parser = util_parser '; end # :category: test' + parser = util_parser 'AAA = 1 # :category: test' directive, value = parser.read_directive %w[category] assert_equal 'category', directive assert_equal 'test', value - assert_kind_of RDoc::RubyToken::TkSEMICOLON, parser.get_tk + assert_equal :on_const, parser.get_tk[:kind] end def test_read_documentation_modifiers @@ -2974,10 +3061,10 @@ end def test_sanity_integer util_parser '1' - assert_equal '1', @parser.get_tk.text + assert_equal '1', @parser.get_tk[:text] util_parser '1.0' - assert_equal '1.0', @parser.get_tk.text + assert_equal '1.0', @parser.get_tk[:text] end def test_sanity_interpolation @@ -2986,7 +3073,7 @@ end while tk = @parser.get_tk do last_tk = tk end - assert_equal "\n", last_tk.text + assert_equal 'end', last_tk[:text] end # If you're writing code like this you're doing it wrong @@ -2994,15 +3081,15 @@ end def test_sanity_interpolation_crazy util_parser '"#{"#{"a")}" if b}"' - assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk.text - assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class + assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk[:text] + assert_equal nil, @parser.get_tk end def test_sanity_interpolation_curly util_parser '%{ #{} }' - assert_equal '%{ #{} }', @parser.get_tk.text - assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class + assert_equal '%{ #{} }', @parser.get_tk[:text] + assert_equal nil, @parser.get_tk end def test_sanity_interpolation_format @@ -3537,6 +3624,53 @@ end assert_equal 2, public_method_count end + def test_scan_constant_visibility + util_parser <<-RUBY +class C + CONST_A = 123 + + CONST_B = 234 + private_constant :CONST_B + + CONST_C = 345 + public_constant :CONST_C +end + RUBY + + @parser.scan + + c = @store.find_class_named 'C' + const_a, const_b, const_c, const_d = c.constants.sort_by(&:name) + + assert_equal 'CONST_A', const_a.name + assert_equal :public, const_a.visibility + + assert_equal 'CONST_B', const_b.name + assert_equal :private, const_b.visibility + + assert_equal 'CONST_C', const_c.name + assert_equal :public, const_c.visibility + end + + def test_document_after_rescue_inside_paren + util_parser <<-RUBY +class C + attr_accessor :sample if (1.inexistent_method rescue false) + # first + # second + def a + end +end + RUBY + + @parser.scan + + c = @store.find_class_named 'C' + + c_a = c.find_method_named 'a' + assert_equal "first\nsecond", c_a.comment.text + end + def test_singleton_method_via_eigenclass util_parser <<-RUBY class C diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb deleted file mode 100644 index 9ab5844b54..0000000000 --- a/test/rdoc/test_rdoc_ruby_lex.rb +++ /dev/null @@ -1,1095 +0,0 @@ -# coding: UTF-8 -# frozen_string_literal: false - -require 'rdoc/test_case' - -class TestRDocRubyLex < RDoc::TestCase - - def setup - @TK = RDoc::RubyToken - end - - def test_token_position - tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil - - assert_equal '[', tokens[0].text - assert_equal 0, tokens[0].seek - assert_equal 1, tokens[0].line_no - assert_equal 0, tokens[0].char_no - assert_equal '1', tokens[2].text - assert_equal 2, tokens[2].seek - assert_equal 1, tokens[2].line_no - assert_equal 2, tokens[2].char_no - assert_equal ':a', tokens[5].text - assert_equal 5, tokens[5].seek - assert_equal 1, tokens[5].line_no - assert_equal 5, tokens[5].char_no - assert_equal 'nil', tokens[8].text - assert_equal 9, tokens[8].seek - assert_equal 1, tokens[8].line_no - assert_equal 9, tokens[8].char_no - assert_equal ']', tokens[10].text - assert_equal 13, tokens[10].seek - assert_equal 1, tokens[10].line_no - assert_equal 13, tokens[10].char_no - end - - def test_class_tokenize - tokens = RDoc::RubyLex.tokenize "def x() end", nil - - expected = [ - @TK::TkDEF .new( 0, 1, 0, "def"), - @TK::TkSPACE .new( 3, 1, 3, " "), - @TK::TkIDENTIFIER.new( 4, 1, 4, "x"), - @TK::TkLPAREN .new( 5, 1, 5, "("), - @TK::TkRPAREN .new( 6, 1, 6, ")"), - @TK::TkSPACE .new( 7, 1, 7, " "), - @TK::TkEND .new( 8, 1, 8, "end"), - @TK::TkNL .new(11, 1, 11, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize___END__ - tokens = RDoc::RubyLex.tokenize '__END__', nil - - expected = [ - @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'), - @TK::TkNL .new(7, 1, 7, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize___ENCODING__ - tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil - - expected = [ - @TK::Tk__ENCODING__.new( 0, 1, 0, '__ENCODING__'), - @TK::TkNL .new(12, 1, 12, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_character_literal - tokens = RDoc::RubyLex.tokenize "?c", nil - - expected = [ - @TK::TkCHAR.new( 0, 1, 0, "?c"), - @TK::TkNL .new( 2, 1, 2, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_character_literal_with_escape - tokens = RDoc::RubyLex.tokenize "?\\s", nil - - expected = [ - @TK::TkCHAR.new( 0, 1, 0, "?\\s"), - @TK::TkNL .new( 3, 1, 3, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_def_heredoc - tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil -def x - < "foo" }', nil - - expected = [ - @TK::TkLBRACE .new( 0, 1, 0, '{'), - @TK::TkSPACE .new( 1, 1, 1, ' '), - @TK::TkSYMBOL .new( 2, 1, 2, ':class'), - @TK::TkSPACE .new( 8, 1, 8, ' '), - @TK::TkHASHROCKET.new( 9, 1, 9, '=>'), - @TK::TkSPACE .new(11, 1, 11, ' '), - @TK::TkSTRING .new(12, 1, 12, '"foo"'), - @TK::TkSPACE .new(17, 1, 17, ' '), - @TK::TkRBRACE .new(18, 1, 18, '}'), - @TK::TkNL .new(19, 1, 19, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_heredoc_CR_NL - tokens = RDoc::RubyLex.tokenize <<-RUBY, nil -string = <<-STRING\r -Line 1\r -Line 2\r - STRING\r - RUBY - - expected = [ - @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), - @TK::TkSPACE .new( 6, 1, 6, ' '), - @TK::TkASSIGN .new( 7, 1, 7, '='), - @TK::TkSPACE .new( 8, 1, 8, ' '), - @TK::TkHEREDOCBEG.new( 9, 1, 9, '<<-STRING'), - @TK::TkSPACE .new(18, 1, 18, "\r"), - @TK::TkNL .new(19, 1, 19, "\n"), - @TK::TkHEREDOC .new(19, 1, 19, - %Q{Line 1\nLine 2\n}), - @TK::TkHEREDOCEND.new(45, 4, 36, " STRING\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_opassign - tokens = RDoc::RubyLex.tokenize <<'RUBY', nil -a %= b -a /= b -a -= b -a += b -a |= b -a &= b -a >>= b -a <<= b -a *= b -a &&= b -a ||= b -a **= b -RUBY - - expected = [ - @TK::TkIDENTIFIER.new( 0, 1, 0, "a"), - @TK::TkSPACE .new( 1, 1, 1, " "), - @TK::TkOPASGN .new( 2, 1, 2, "%"), - @TK::TkSPACE .new( 4, 1, 4, " "), - @TK::TkIDENTIFIER.new( 5, 1, 5, "b"), - @TK::TkNL .new( 6, 1, 6, "\n"), - @TK::TkIDENTIFIER.new( 7, 2, 0, "a"), - @TK::TkSPACE .new( 8, 2, 1, " "), - @TK::TkOPASGN .new( 9, 2, 2, "/"), - @TK::TkSPACE .new( 11, 2, 4, " "), - @TK::TkIDENTIFIER.new( 12, 2, 5, "b"), - @TK::TkNL .new( 13, 2, 7, "\n"), - @TK::TkIDENTIFIER.new( 14, 3, 0, "a"), - @TK::TkSPACE .new( 15, 3, 1, " "), - @TK::TkOPASGN .new( 16, 3, 2, "-"), - @TK::TkSPACE .new( 18, 3, 4, " "), - @TK::TkIDENTIFIER.new( 19, 3, 5, "b"), - @TK::TkNL .new( 20, 3, 14, "\n"), - @TK::TkIDENTIFIER.new( 21, 4, 0, "a"), - @TK::TkSPACE .new( 22, 4, 1, " "), - @TK::TkOPASGN .new( 23, 4, 2, "+"), - @TK::TkSPACE .new( 25, 4, 4, " "), - @TK::TkIDENTIFIER.new( 26, 4, 5, "b"), - @TK::TkNL .new( 27, 4, 21, "\n"), - @TK::TkIDENTIFIER.new( 28, 5, 0, "a"), - @TK::TkSPACE .new( 29, 5, 1, " "), - @TK::TkOPASGN .new( 30, 5, 2, "|"), - @TK::TkSPACE .new( 32, 5, 4, " "), - @TK::TkIDENTIFIER.new( 33, 5, 5, "b"), - @TK::TkNL .new( 34, 5, 28, "\n"), - @TK::TkIDENTIFIER.new( 35, 6, 0, "a"), - @TK::TkSPACE .new( 36, 6, 1, " "), - @TK::TkOPASGN .new( 37, 6, 2, "&"), - @TK::TkSPACE .new( 39, 6, 4, " "), - @TK::TkIDENTIFIER.new( 40, 6, 5, "b"), - @TK::TkNL .new( 41, 6, 35, "\n"), - @TK::TkIDENTIFIER.new( 42, 7, 0, "a"), - @TK::TkSPACE .new( 43, 7, 1, " "), - @TK::TkOPASGN .new( 44, 7, 2, ">>"), - @TK::TkSPACE .new( 47, 7, 5, " "), - @TK::TkIDENTIFIER.new( 48, 7, 6, "b"), - @TK::TkNL .new( 49, 7, 42, "\n"), - @TK::TkIDENTIFIER.new( 50, 8, 0, "a"), - @TK::TkSPACE .new( 51, 8, 1, " "), - @TK::TkOPASGN .new( 52, 8, 2, "<<"), - @TK::TkSPACE .new( 55, 8, 5, " "), - @TK::TkIDENTIFIER.new( 56, 8, 6, "b"), - @TK::TkNL .new( 57, 8, 50, "\n"), - @TK::TkIDENTIFIER.new( 58, 9, 0, "a"), - @TK::TkSPACE .new( 59, 9, 1, " "), - @TK::TkOPASGN .new( 60, 9, 2, "*"), - @TK::TkSPACE .new( 62, 9, 4, " "), - @TK::TkIDENTIFIER.new( 63, 9, 5, "b"), - @TK::TkNL .new( 64, 9, 58, "\n"), - @TK::TkIDENTIFIER.new( 65, 10, 0, "a"), - @TK::TkSPACE .new( 66, 10, 1, " "), - @TK::TkOPASGN .new( 67, 10, 2, "&&"), - @TK::TkSPACE .new( 70, 10, 5, " "), - @TK::TkIDENTIFIER.new( 71, 10, 6, "b"), - @TK::TkNL .new( 72, 10, 65, "\n"), - @TK::TkIDENTIFIER.new( 73, 11, 0, "a"), - @TK::TkSPACE .new( 74, 11, 1, " "), - @TK::TkOPASGN .new( 75, 11, 2, "||"), - @TK::TkSPACE .new( 78, 11, 5, " "), - @TK::TkIDENTIFIER.new( 79, 11, 6, "b"), - @TK::TkNL .new( 80, 11, 73, "\n"), - @TK::TkIDENTIFIER.new( 81, 12, 0, "a"), - @TK::TkSPACE .new( 82, 12, 1, " "), - @TK::TkOPASGN .new( 83, 12, 2, "**"), - @TK::TkSPACE .new( 86, 12, 5, " "), - @TK::TkIDENTIFIER.new( 87, 12, 6, "b"), - @TK::TkNL .new( 88, 12, 81, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_heredoc_call - tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil -string = <<-STRING.chomp -Line 1 -Line 2 - STRING - RUBY - - expected = [ - @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), - @TK::TkSPACE .new( 6, 1, 6, ' '), - @TK::TkASSIGN .new( 7, 1, 7, '='), - @TK::TkSPACE .new( 8, 1, 8, ' '), - @TK::TkHEREDOCBEG.new( 9, 1, 9, '<<-STRING'), - @TK::TkDOT .new(18, 1, 18, '.'), - @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'), - @TK::TkNL .new(24, 1, 24, "\n"), - @TK::TkHEREDOC .new(24, 1, 24, "Line 1\nLine 2\n"), - @TK::TkHEREDOCEND.new(47, 4, 39, " STRING\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_heredoc_indent - tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil -string = <<-STRING -Line 1 -Line 2 - STRING - RUBY - - expected = [ - @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), - @TK::TkSPACE .new( 6, 1, 6, ' '), - @TK::TkASSIGN .new( 7, 1, 7, '='), - @TK::TkSPACE .new( 8, 1, 8, ' '), - - - @TK::TkHEREDOCBEG.new( 9, 1, 9, '<<-STRING'), - @TK::TkNL .new(18, 1, 18, "\n"), - @TK::TkHEREDOC .new(18, 1, 18, "Line 1\nLine 2\n"), - @TK::TkHEREDOCEND.new(41, 4, 33, " STRING\n") - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_heredoc_missing_end - e = assert_raises RDoc::RubyLex::Error do - RDoc::RubyLex.tokenize <<-'RUBY', nil ->> string1 = <<-TXT ->" That's swell ->" TXT - RUBY - end - - assert_equal 'Missing terminating TXT for string', e.message - end - - def test_class_tokenize_heredoc_percent_N - tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil -a b <<-U -%N -U - RUBY - - expected = [ - @TK::TkIDENTIFIER.new( 0, 1, 0, 'a'), - @TK::TkSPACE .new( 1, 1, 1, ' '), - @TK::TkIDENTIFIER.new( 2, 1, 2, 'b'), - @TK::TkSPACE .new( 3, 1, 3, ' '), - @TK::TkHEREDOCBEG.new( 4, 1, 4, '<<-U'), - @TK::TkNL .new( 8, 1, 8, "\n"), - @TK::TkHEREDOC .new( 8, 1, 8, "%N\n"), - @TK::TkHEREDOCEND.new(13, 3, 12, "U\n") - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_identifier_high_unicode - tokens = RDoc::RubyLex.tokenize '𝖒', nil - - expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒') - - assert_equal expected, tokens.first - end - - def test_class_tokenize_lambda - tokens = RDoc::RubyLex.tokenize 'a = -> x, y { x + y }', nil - - expected = [ - @TK::TkIDENTIFIER.new( 0, 1, 0, 'a'), - @TK::TkSPACE .new( 1, 1, 1, ' '), - @TK::TkASSIGN .new( 2, 1, 2, '='), - @TK::TkSPACE .new( 3, 1, 3, ' '), - @TK::TkLAMBDA .new( 4, 1, 4, '->'), - @TK::TkSPACE .new( 6, 1, 6, ' '), - @TK::TkIDENTIFIER.new( 7, 1, 7, 'x'), - @TK::TkCOMMA .new( 8, 1, 8, ','), - @TK::TkSPACE .new( 9, 1, 9, ' '), - @TK::TkIDENTIFIER.new(10, 1, 10, 'y'), - @TK::TkSPACE .new(11, 1, 11, ' '), - @TK::TkfLBRACE .new(12, 1, 12, '{'), - @TK::TkSPACE .new(13, 1, 13, ' '), - @TK::TkIDENTIFIER.new(14, 1, 14, 'x'), - @TK::TkSPACE .new(15, 1, 15, ' '), - @TK::TkPLUS .new(16, 1, 16, '+'), - @TK::TkSPACE .new(17, 1, 17, ' '), - @TK::TkIDENTIFIER.new(18, 1, 18, 'y'), - @TK::TkSPACE .new(19, 1, 19, ' '), - @TK::TkRBRACE .new(20, 1, 20, '}'), - @TK::TkNL .new(21, 1, 21, "\n") - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_1 - tokens = RDoc::RubyLex.tokenize 'v%10==10', nil - - expected = [ - @TK::TkIDENTIFIER.new(0, 1, 0, 'v'), - @TK::TkMOD.new( 1, 1, 1, '%'), - @TK::TkINTEGER.new( 2, 1, 2, '10'), - @TK::TkEQ.new( 4, 1, 4, '=='), - @TK::TkINTEGER.new( 6, 1, 6, '10'), - @TK::TkNL.new( 8, 1, 8, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_r - tokens = RDoc::RubyLex.tokenize '%r[hi]', nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, '%r[hi]'), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_r_with_slash - tokens = RDoc::RubyLex.tokenize '%r/hi/', nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, '%r/hi/'), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_large_q - tokens = RDoc::RubyLex.tokenize '%Q/hi/', nil - - expected = [ - @TK::TkSTRING.new( 0, 1, 0, '%Q/hi/'), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_large_q_with_double_quote - tokens = RDoc::RubyLex.tokenize '%Q"hi"', nil - - expected = [ - @TK::TkSTRING.new( 0, 1, 0, '%Q"hi"'), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_w - tokens = RDoc::RubyLex.tokenize '%w[hi]', nil - - expected = [ - @TK::TkDSTRING.new( 0, 1, 0, '%w[hi]'), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_w_quote - tokens = RDoc::RubyLex.tokenize '%w"hi"', nil - - expected = [ - @TK::TkDSTRING.new( 0, 1, 0, '%w"hi"'), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_hash_rocket - tokens = RDoc::RubyLex.tokenize "{ :foo=> 1 }", nil - - expected = [ - @TK::TkLBRACE .new( 0, 1, 0, '{'), - @TK::TkSPACE .new( 1, 1, 1, ' '), - @TK::TkSYMBOL .new( 2, 1, 2, ':foo'), - @TK::TkHASHROCKET.new( 6, 1, 6, '=>'), - @TK::TkSPACE .new( 8, 1, 8, ' '), - @TK::TkINTEGER .new( 9, 1, 9, '1'), - @TK::TkSPACE .new(10, 1, 10, ' '), - @TK::TkRBRACE .new(11, 1, 11, '}'), - @TK::TkNL .new(12, 1, 12, "\n") - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_percent_sign_quote - tokens = RDoc::RubyLex.tokenize '%%hi%', nil - - expected = [ - @TK::TkSTRING.new( 0, 1, 0, '%%hi%'), - @TK::TkNL .new( 5, 1, 5, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_regexp - tokens = RDoc::RubyLex.tokenize "/hay/", nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, "/hay/"), - @TK::TkNL .new( 5, 1, 5, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_regexp_options - tokens = RDoc::RubyLex.tokenize "/hAY/i", nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, "/hAY/i"), - @TK::TkNL .new( 6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - - tokens = RDoc::RubyLex.tokenize "/hAY/ix", nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, "/hAY/ix"), - @TK::TkNL .new( 7, 1, 7, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_regexp_backref - tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, "/[csh](..) [csh]\\1 in/"), - @TK::TkNL .new(22, 1, 22, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_regexp_escape - tokens = RDoc::RubyLex.tokenize "/\\//", nil - - expected = [ - @TK::TkREGEXP.new( 0, 1, 0, "/\\//"), - @TK::TkNL .new( 4, 1, 4, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_number_with_sign_character - tokens = RDoc::RubyLex.tokenize "+3--3r", nil - - expected = [ - @TK::TkINTEGER .new(0, 1, 0, "+3"), - @TK::TkMINUS .new(2, 1, 2, "-"), - @TK::TkRATIONAL.new(3, 1, 3, "-3r"), - @TK::TkNL .new(6, 1, 6, "\n"), - ] - - assert_equal expected, tokens - end - - def test_class_tokenize_regexp_continuing_backslash - tokens = RDoc::RubyLex.tokenize "/(? 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' }, + { :line_no => 0, :char_no => 0, :kind => :on_kw, :text => 'KW' }, + { :line_no => 0, :char_no => 0, :kind => :on_ivar, :text => 'IVAR' }, + { :line_no => 0, :char_no => 0, :kind => :on_op, :text => 'Op' }, + { :line_no => 0, :char_no => 0, :kind => :on_ident, :text => 'Id' }, + { :line_no => 0, :char_no => 0, :kind => :on_backref, :text => 'Node' }, + { :line_no => 0, :char_no => 0, :kind => :on_comment, :text => 'COMMENT' }, + { :line_no => 0, :char_no => 0, :kind => :on_regexp, :text => 'REGEXP' }, + { :line_no => 0, :char_no => 0, :kind => :on_tstring, :text => 'STRING' }, + { :line_no => 0, :char_no => 0, :kind => :on_int, :text => 'Val' }, + { :line_no => 0, :char_no => 0, :kind => :on_unknown, :text => '\\' } ] expected = [