* lib/rdoc/cross_reference.rb: Fixed matching of C#=== or #===. RDoc

bug #164
* test/rdoc/test_rdoc_cross_reference.rb:  Test for above.

* lib/rdoc/parser/changelog.rb:  Fixed parsing of dates.  RDoc bug #165
* test/rdoc/test_rdoc_parser_changelog.rb:  Test for above.

* lib/rdoc/parser.rb:  Fixed parsing multibyte files with incomplete
  characters at byte 1024.  [ruby-trunk - Bug #6393]
  Fixed handling of -E.  [ruby-trunk - Bug #6392]
* test/rdoc/test_rdoc_options.rb:  Test for above.
* test/rdoc/test_rdoc_parser.rb:  ditto.
* test/rdoc/test_rdoc_parser_c.rb:  ditto.
* test/rdoc/test_rdoc_parser_changelog.rb:  ditto.
* test/rdoc/test_rdoc_parser_markdown.rb:  ditto.
* test/rdoc/test_rdoc_parser_rd.rb:  ditto.
* test/rdoc/test_rdoc_rdoc.rb:  ditto.

* lib/rdoc/tom_doc.rb:  Fixed parsing of [] in TomDoc arguments list.
  RDoc bug #167
* test/rdoc/test_rdoc_tom_doc.rb:  Test for above.

* lib/rdoc.rb:  Update version.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38690 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,29 @@
+Fri Jan  4 15:05:25 2013  Eric Hodel  <drbrain@segment7.net>
+
+	* lib/rdoc/cross_reference.rb:  Fixed matching of C#=== or #===.  RDoc
+	  bug #164
+	* test/rdoc/test_rdoc_cross_reference.rb:  Test for above.
+
+	* lib/rdoc/parser/changelog.rb:  Fixed parsing of dates.  RDoc bug #165
+	* test/rdoc/test_rdoc_parser_changelog.rb:  Test for above.
+
+	* lib/rdoc/parser.rb:  Fixed parsing multibyte files with incomplete
+	  characters at byte 1024.  [ruby-trunk - Bug #6393]
+	  Fixed handling of -E.  [ruby-trunk - Bug #6392]
+	* test/rdoc/test_rdoc_options.rb:  Test for above.
+	* test/rdoc/test_rdoc_parser.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_c.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_changelog.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_markdown.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_rd.rb:  ditto.
+	* test/rdoc/test_rdoc_rdoc.rb:  ditto.
+
+	* lib/rdoc/tom_doc.rb:  Fixed parsing of [] in TomDoc arguments list.
+	  RDoc bug #167
+	* test/rdoc/test_rdoc_tom_doc.rb:  Test for above.
+
+	* lib/rdoc.rb:  Update version.
+
 Fri Jan  4 11:51:00 2013  Zachary Scott  <zachary@zacharyscott.net>
 
 	* lib/forwardable.rb: Fix rdoc parameters for ::def_single_delegator.

lib/rdoc.rb

@@ -64,7 +64,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '4.0.0.preview2.1'
+  VERSION = '4.0.0.preview3.1'
 
   ##
   # Method visibilities

lib/rdoc/cross_reference.rb

@@ -18,7 +18,7 @@ class RDoc::CrossReference
   #
   # See CLASS_REGEXP_STR
 
-  METHOD_REGEXP_STR = '([a-z]\w*[!?=]?|%)(?:\([\w.+*/=<>-]*\))?'
+  METHOD_REGEXP_STR = '([a-z]\w*[!?=]?|%|===)(?:\([\w.+*/=<>-]*\))?'
 
   ##
   # Regular expressions matching text that should potentially have

lib/rdoc/parser.rb

@@ -58,7 +58,7 @@ class RDoc::Parser
     old_ext = old_ext.sub(/^\.(.*)/, '\1')
     new_ext = new_ext.sub(/^\.(.*)/, '\1')
 
-    parser = can_parse "xxx.#{old_ext}"
+    parser = can_parse_by_name "xxx.#{old_ext}"
     return false unless parser
 
     RDoc::Parser.parsers.unshift [/\.#{new_ext}$/, parser]
@@ -77,14 +77,14 @@ class RDoc::Parser
 
     have_encoding = s.respond_to? :encoding
 
-    if have_encoding then
-      return false if s.encoding != Encoding::ASCII_8BIT and s.valid_encoding?
-    end
-
     return true if s[0, 2] == Marshal.dump('')[0, 2] or s.index("\x00")
 
     if have_encoding then
-      s.force_encoding Encoding.default_external
+      mode = "r"
+      s.sub!(/\A#!.*\n/, '')     # assume shebang line isn't longer than 1024.
+      encoding = s[/^\s*\#\s*(?:-\*-\s*)?(?:en)?coding:\s*([^\s;]+?)(?:-\*-|[\s;])/, 1]
+      mode = "r:#{encoding}" if encoding
+      s = File.open(file, mode) {|f| f.gets(nil, 1024)}
 
       not s.valid_encoding?
     else
@@ -131,23 +131,36 @@ class RDoc::Parser
     zip_signature == "PK\x03\x04" or
       zip_signature == "PK\x05\x06" or
       zip_signature == "PK\x07\x08"
+  rescue
+    false
   end
 
   ##
   # Return a parser that can handle a particular extension
 
-  def self.can_parse(file_name)
+  def self.can_parse file_name
-    parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }.last
+    parser = can_parse_by_name file_name
 
     # HACK Selenium hides a jar file using a .txt extension
     return if parser == RDoc::Parser::Simple and zip? file_name
 
+    parser
+  end
+
+  ##
+  # Returns a parser that can handle the extension for +file_name+.  This does
+  # not depend upon the file being readable.
+
+  def self.can_parse_by_name file_name
+    _, parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }
+
     # The default parser must not parse binary files
     ext_name = File.extname file_name
     return parser if ext_name.empty?
+
     if parser == RDoc::Parser::Simple and ext_name !~ /txt|rdoc/ then
       case check_modeline file_name
-      when 'rdoc' then # continue
+      when nil, 'rdoc' then # continue
       else return nil
       end
     end
@@ -173,6 +186,8 @@ class RDoc::Parser
       type = $1
     end
 
+    return nil if /coding:/i =~ type
+
     type.downcase
   rescue ArgumentError # invalid byte sequence, etc.
   end
@@ -204,6 +219,8 @@ class RDoc::Parser
     return unless parser
 
     parser.new top_level, file_name, content, options, stats
+  rescue SystemCallError
+    nil
   end
 
   ##

lib/rdoc/parser/changelog.rb

@@ -102,7 +102,12 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
 
   def group_entries entries
     entries.group_by do |title, _|
-      Time.parse(title).strftime "%Y-%m-%d"
+      begin
+        Time.parse(title).strftime '%Y-%m-%d'
+      rescue NoMethodError, ArgumentError
+        time, = title.split '  ', 2
+        Time.parse(time).strftime '%Y-%m-%d'
+      end
     end
   end
 
@@ -139,6 +144,9 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
           time = Time.parse entry_name
           # HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other"
           entry_name = nil unless entry_name =~ /#{time.year}/
+        rescue NoMethodError
+          time, = entry_name.split '  ', 2
+          time = Time.parse time
         rescue ArgumentError
           entry_name = nil
         end

lib/rdoc/tom_doc.rb

@@ -218,7 +218,7 @@ class RDoc::TomDoc < RDoc::Markup::Parser
                    @tokens << [:HEADER, 3, *token_pos(pos)]
 
                    [:TEXT, @s[1], *token_pos(pos)]
-                 when @s.scan(/([:\w]\w*)[ ]+- /) then
+                 when @s.scan(/([:\w][\w\[\]]*)[ ]+- /) then
                    [:NOTE, @s[1], *token_pos(pos)]
                  else
                    @s.scan(/.*/)

test/rdoc/test.ja.largedoc

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+ 吾輩（わがはい）は猫である。名前はまだ無い。
+ どこで生れたかとんと見当（けんとう）がつかぬ。何でも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している。吾輩はここで始めて人間というものを見た。しかもあとで聞くとそれは書生という人間中で一番獰悪（どうあく）な種族であったそうだ。この書生というのは時々我々を捕（つかま）えて煮（に）て食うという話である。しかしその当時は何という考もなかったから別段恐しいとも思わなかった。ただ彼の掌（てのひら）に載せられてスーと持ち上げられた時何だかフワフワした感じがあったばかりである。掌の上で少し落ちついて書生の顔を見たのがいわゆる人間というものの見始（みはじめ）であろう。この時妙なものだと思った感じが今でも残っている。第一毛をもって装飾されべきはずの顔がつるつるしてまるで薬缶（やかん）だ。その後（ご）猫にもだいぶ逢（あ）ったがこんな片輪（かたわ）には一度も出会（でく）わした事がない。のみならず顔の真中があまりに突起している。そうしてその穴の中から時々ぷうぷうと煙（けむり）を吹く。どうも咽（む）せぽくて実に弱った。これが人間の飲む煙草（たばこ）というものである事はようやくこの頃知った。

test/rdoc/test_rdoc_cross_reference.rb

@@ -16,6 +16,14 @@ class TestRDocCrossReference < XrefTestCase
     assert_equal name, @xref.resolve(name, name)
   end
 
+  def test_METHOD_REGEXP_STR
+    re = /#{RDoc::CrossReference::METHOD_REGEXP_STR}/
+
+    re =~ '==='
+
+    assert_equal '===', $&
+  end
+
   def test_resolve_C2
     @xref = RDoc::CrossReference.new @c2
 
@@ -129,6 +137,13 @@ class TestRDocCrossReference < XrefTestCase
     assert_ref @c2_c3_m, '::C2::C3#m(*)'
   end
 
+  def test_resolve_method_equals3
+    m = RDoc::AnyMethod.new '', '==='
+    @c1.add_method m
+
+    assert_ref m, '==='
+  end
+
   def test_resolve_page
     page = @store.add_file 'README.txt'
     page.parser = RDoc::Parser::Simple

test/rdoc/test_rdoc_options.rb

@@ -549,6 +549,17 @@ rdoc_include:
     FileUtils.rm_rf tmpdir
   end
 
+  def test_parse_extension_alias
+    out, err = capture_io do
+      @options.parse %w[--extension foobar=rdoc]
+    end
+
+    assert_includes RDoc::Parser.parsers, [/\.foobar$/, RDoc::Parser::Simple]
+
+    assert_empty out
+    assert_empty err
+  end
+
   def test_setup_generator
     test_generator = Class.new do
       def self.setup_options op

test/rdoc/test_rdoc_parser.rb

@@ -33,8 +33,18 @@ class TestRDocParser < RDoc::TestCase
   end
 
   def test_class_binary_large_japanese_rdoc
-    file_name = File.expand_path '../test.ja.large.rdoc', __FILE__
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
-    assert !@RP.binary?(file_name)
+
+    capture_io do
+      begin
+        extenc, Encoding.default_external =
+          Encoding.default_external, Encoding::US_ASCII
+        file_name = File.expand_path '../test.ja.largedoc', __FILE__
+        assert !@RP.binary?(file_name)
+      ensure
+        Encoding.default_external = extenc
+      end
+    end
   end
 
   def test_class_binary_japanese_rdoc
@@ -51,7 +61,7 @@ class TestRDocParser < RDoc::TestCase
 
     assert_equal @RP::Simple, @RP.can_parse(readme_file_name)
 
-    assert_nil @RP.can_parse(@binary_dat)
+    assert_equal @RP::Simple, @RP.can_parse(@binary_dat)
 
     jtest_file_name = File.expand_path '../test.ja.txt', __FILE__
     assert_equal @RP::Simple, @RP.can_parse(jtest_file_name)
@@ -61,20 +71,12 @@ class TestRDocParser < RDoc::TestCase
 
     readme_file_name = File.expand_path '../README', __FILE__
     assert_equal @RP::Simple, @RP.can_parse(readme_file_name)
-  end
 
-  def test_class_can_parse_forbidden
+    jtest_largerdoc_file_name = File.expand_path '../test.ja.largedoc', __FILE__
-    skip 'chmod not supported' if Gem.win_platform?
+    assert_equal @RP::Simple, @RP.can_parse(jtest_largerdoc_file_name)
 
-    Tempfile.open 'forbidden' do |io|
+    @RP.alias_extension 'rdoc', 'largedoc'
-      begin
+    assert_equal @RP::Simple, @RP.can_parse(jtest_largerdoc_file_name)
-        File.chmod 0000, io.path
-
-        assert_nil @RP.can_parse io.path
-      ensure
-        File.chmod 0400, io.path
-      end
-    end
   end
 
   def test_class_for_executable
@@ -82,6 +84,7 @@ class TestRDocParser < RDoc::TestCase
       content = "#!/usr/bin/env ruby -w\n"
       open 'app', 'w' do |io| io.write content end
       app = @store.add_file 'app'
+
       parser = @RP.for app, 'app', content, @options, :stats
 
       assert_kind_of RDoc::Parser::Ruby, parser
@@ -90,6 +93,23 @@ class TestRDocParser < RDoc::TestCase
     end
   end
 
+  def test_class_for_forbidden
+    skip 'chmod not supported' if Gem.win_platform?
+
+    Tempfile.open 'forbidden' do |io|
+      begin
+        File.chmod 0000, io.path
+        forbidden = @store.add_file io.path
+
+        parser = @RP.for forbidden, 'forbidden', '', @options, :stats
+
+        assert_nil parser
+      ensure
+        File.chmod 0400, io.path
+      end
+    end
+  end
+
   def test_can_parse_modeline
     readme_ext = File.join Dir.tmpdir, "README.EXT.#{$$}"
 
@@ -126,6 +146,18 @@ class TestRDocParser < RDoc::TestCase
     File.unlink readme_ext
   end
 
+  def test_check_modeline_coding
+    readme_ext = File.join Dir.tmpdir, "README.EXT.#{$$}"
+
+    open readme_ext, 'w' do |io|
+      io.puts "# -*- coding: utf-8 -*-"
+    end
+
+    assert_nil @RP.check_modeline readme_ext
+  ensure
+    File.unlink readme_ext
+  end
+
   def test_check_modeline_with_other
     readme_ext = File.join Dir.tmpdir, "README.EXT.#{$$}"
 

test/rdoc/test_rdoc_parser_c.rb

@@ -64,17 +64,40 @@ class TestRDocParserC < RDoc::TestCase
   def test_class_can_parse
     c_parser = RDoc::Parser::C
 
-    assert_equal c_parser, c_parser.can_parse('file.C')
+    temp_dir do
-    assert_equal c_parser, c_parser.can_parse('file.CC')
+      FileUtils.touch 'file.C'
-    assert_equal c_parser, c_parser.can_parse('file.H')
+      assert_equal c_parser, c_parser.can_parse('file.C')
-    assert_equal c_parser, c_parser.can_parse('file.HH')
+
-    assert_equal c_parser, c_parser.can_parse('file.c')
+      FileUtils.touch 'file.CC'
-    assert_equal c_parser, c_parser.can_parse('file.cc')
+      assert_equal c_parser, c_parser.can_parse('file.CC')
-    assert_equal c_parser, c_parser.can_parse('file.cpp')
+
-    assert_equal c_parser, c_parser.can_parse('file.cxx')
+      FileUtils.touch 'file.H'
-    assert_equal c_parser, c_parser.can_parse('file.h')
+      assert_equal c_parser, c_parser.can_parse('file.H')
-    assert_equal c_parser, c_parser.can_parse('file.hh')
+
-    assert_equal c_parser, c_parser.can_parse('file.y')
+      FileUtils.touch 'file.HH'
+      assert_equal c_parser, c_parser.can_parse('file.HH')
+
+      FileUtils.touch 'file.c'
+      assert_equal c_parser, c_parser.can_parse('file.c')
+
+      FileUtils.touch 'file.cc'
+      assert_equal c_parser, c_parser.can_parse('file.cc')
+
+      FileUtils.touch 'file.cpp'
+      assert_equal c_parser, c_parser.can_parse('file.cpp')
+
+      FileUtils.touch 'file.cxx'
+      assert_equal c_parser, c_parser.can_parse('file.cxx')
+
+      FileUtils.touch 'file.h'
+      assert_equal c_parser, c_parser.can_parse('file.h')
+
+      FileUtils.touch 'file.hh'
+      assert_equal c_parser, c_parser.can_parse('file.hh')
+
+      FileUtils.touch 'file.y'
+      assert_equal c_parser, c_parser.can_parse('file.y')
+    end
   end
 
   def test_initialize

test/rdoc/test_rdoc_parser_changelog.rb

@@ -25,11 +25,15 @@ class TestRDocParserChangeLog < RDoc::TestCase
   def test_class_can_parse
     parser = RDoc::Parser::ChangeLog
 
-    assert_equal parser, parser.can_parse('ChangeLog')
+    temp_dir do
+      FileUtils.touch 'ChangeLog'
+      assert_equal parser, parser.can_parse('ChangeLog')
 
-    assert_equal parser, parser.can_parse(@tempfile.path)
+      assert_equal parser, parser.can_parse(@tempfile.path)
 
-    assert_equal RDoc::Parser::Ruby, parser.can_parse('ChangeLog.rb')
+      FileUtils.touch 'ChangeLog.rb'
+      assert_equal RDoc::Parser::Ruby, parser.can_parse('ChangeLog.rb')
+    end
   end
 
   def test_continue_entry_body
@@ -178,7 +182,9 @@ class TestRDocParserChangeLog < RDoc::TestCase
       [ 'Tue Dec  4 08:32:10 2012  Eric Hodel  <drbrain@segment7.net>',
         %w[three four]],
       [ 'Mon Dec  3 20:28:02 2012  Koichi Sasada  <ko1@atdot.net>',
-        %w[five six]]]
+        %w[five six]],
+      [ '2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>',
+        %w[seven eight]]]
 
     expected = {
       '2012-12-04' => [
@@ -189,6 +195,9 @@ class TestRDocParserChangeLog < RDoc::TestCase
       '2012-12-03' => [
         ['Mon Dec  3 20:28:02 2012  Koichi Sasada  <ko1@atdot.net>',
           %w[five six]]],
+      '2008-01-30' => [
+        ['2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>',
+          %w[seven eight]]],
     }
 
     assert_equal expected, parser.group_entries(entries)
@@ -222,6 +231,25 @@ Other note that will be ignored
     assert_equal expected, parser.parse_entries
   end
 
+  def test_parse_entries_bad_time
+    parser = util_parser <<-ChangeLog
+2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>
+
+        PR libffi/34612
+        * src/x86/sysv.S (ffi_closure_SYSV): Pop 4 byte from stack when
+        returning struct.
+
+    ChangeLog
+
+    expected = [
+      [ '2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>',
+        [ 'src/x86/sysv.S (ffi_closure_SYSV): Pop 4 byte from stack when ' +
+          'returning struct.']]
+    ]
+
+    assert_equal expected, parser.parse_entries
+  end
+
   def test_parse_entries_gnu
     parser = util_parser <<-ChangeLog
 1998-08-17  Richard Stallman  <rms@gnu.org>

test/rdoc/test_rdoc_parser_markdown.rb

@@ -27,11 +27,17 @@ class TestRDocParserMarkdown < RDoc::TestCase
   end
 
   def test_class_can_parse
-    assert_equal @RP::Markdown, @RP.can_parse('foo.md')
+    temp_dir do
-    assert_equal @RP::Markdown, @RP.can_parse('foo.md.ja')
+      FileUtils.touch 'foo.md'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.md')
+      FileUtils.touch 'foo.md.ja'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.md.ja')
 
-    assert_equal @RP::Markdown, @RP.can_parse('foo.markdown')
+      FileUtils.touch 'foo.markdown'
-    assert_equal @RP::Markdown, @RP.can_parse('foo.markdown.ja')
+      assert_equal @RP::Markdown, @RP.can_parse('foo.markdown')
+      FileUtils.touch 'foo.markdown.ja'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.markdown.ja')
+    end
   end
 
   def test_scan

test/rdoc/test_rdoc_parser_rd.rb

@@ -34,8 +34,13 @@ class TestRDocParserRd < RDoc::TestCase
   end
 
   def test_class_can_parse
-    assert_equal @RP::RD, @RP.can_parse('foo.rd')
+    temp_dir do
-    assert_equal @RP::RD, @RP.can_parse('foo.rd.ja')
+      FileUtils.touch 'foo.rd'
+      assert_equal @RP::RD, @RP.can_parse('foo.rd')
+
+      FileUtils.touch 'foo.rd.ja'
+      assert_equal @RP::RD, @RP.can_parse('foo.rd.ja')
+    end
   end
 
   def test_scan

test/rdoc/test_rdoc_rdoc.rb

@@ -14,19 +14,22 @@ class TestRDocRDoc < RDoc::TestCase
 
   def test_document # functional test
     options = RDoc::Options.new
-    options.files = [File.expand_path('../xref_data.rb')]
+    options.files = [File.expand_path('../xref_data.rb', __FILE__)]
     options.setup_generator 'ri'
     options.main_page = 'MAIN_PAGE.rdoc'
+    options.root      = Pathname File.expand_path('..', __FILE__)
     options.title     = 'title'
 
     rdoc = RDoc::RDoc.new
 
     temp_dir do
+      options.op_dir = 'ri'
+
       capture_io do
         rdoc.document options
       end
 
-      assert File.directory? 'doc'
+      assert File.directory? 'ri'
       assert_equal rdoc, rdoc.store.rdoc
     end
 

test/rdoc/test_rdoc_tom_doc.rb

@@ -156,6 +156,24 @@ description - arguments description
     assert_equal expected, @TD.parse(text)
   end
 
+  def test_parse_arguments_array
+    text = <<-TEXT
+Create new Arg object.
+
+names[] - names of arguments
+    TEXT
+
+    expected =
+      doc(
+        para('Create new Arg object.'),
+        blank_line,
+        list(:NOTE,
+          item(%w[names[]],
+            para('names of arguments'))))
+
+    assert_equal expected, @TD.parse(text)
+  end
+
   def test_parse_arguments_multiline
     text = <<-TEXT
 Do some stuff
@@ -343,6 +361,25 @@ description - arguments description
     assert_equal expected, @td.tokens
   end
 
+  def test_tokenize_arguments_array
+    @td.tokenize <<-TEXT
+Create new Arg object.
+
+names[stuff] - names of arguments
+    TEXT
+
+    expected = [
+      [:TEXT,    "Create new Arg object.",  0, 0],
+      [:NEWLINE, "\n",                     22, 0],
+      [:NEWLINE, "\n",                      0, 1],
+      [:NOTE,    "names[stuff]",            0, 2],
+      [:TEXT,    "names of arguments",     15, 2],
+      [:NEWLINE, "\n",                     33, 2],
+    ]
+
+    assert_equal expected, @td.tokens
+  end
+
   def test_tokenize_arguments_multiline
     @td.tokenize <<-TEXT
 Do some stuff