* lib/rdoc/cross_reference.rb: Fixed matching of C#=== or #===. RDoc

bug #164 * test/rdoc/test_rdoc_cross_reference.rb: Test for above. * lib/rdoc/parser/changelog.rb: Fixed parsing of dates. RDoc bug #165 * test/rdoc/test_rdoc_parser_changelog.rb: Test for above. * lib/rdoc/parser.rb: Fixed parsing multibyte files with incomplete characters at byte 1024. [ruby-trunk - Bug #6393] Fixed handling of -E. [ruby-trunk - Bug #6392] * test/rdoc/test_rdoc_options.rb: Test for above. * test/rdoc/test_rdoc_parser.rb: ditto. * test/rdoc/test_rdoc_parser_c.rb: ditto. * test/rdoc/test_rdoc_parser_changelog.rb: ditto. * test/rdoc/test_rdoc_parser_markdown.rb: ditto. * test/rdoc/test_rdoc_parser_rd.rb: ditto. * test/rdoc/test_rdoc_rdoc.rb: ditto. * lib/rdoc/tom_doc.rb: Fixed parsing of [] in TomDoc arguments list. RDoc bug #167 * test/rdoc/test_rdoc_tom_doc.rb: Test for above. * lib/rdoc.rb: Update version. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38690 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
2022-11-09 12:17:21 -05:00 · 2013-01-04 06:16:13 +00:00 · 2013-01-04 06:16:13 +00:00 · f3b24b5b7f
commit f3b24b5b7f
parent 5a7c4d2a78
16 changed files with 265 additions and 51 deletions
--- a/26
+++ b/26
@ -1,3 +1,29 @@
+Fri Jan  4 15:05:25 2013  Eric Hodel  <drbrain@segment7.net>
+
+	* lib/rdoc/cross_reference.rb:  Fixed matching of C#=== or #===.  RDoc
+	  bug #164
+	* test/rdoc/test_rdoc_cross_reference.rb:  Test for above.
+
+	* lib/rdoc/parser/changelog.rb:  Fixed parsing of dates.  RDoc bug #165
+	* test/rdoc/test_rdoc_parser_changelog.rb:  Test for above.
+
+	* lib/rdoc/parser.rb:  Fixed parsing multibyte files with incomplete
+	  characters at byte 1024.  [ruby-trunk - Bug #6393]
+	  Fixed handling of -E.  [ruby-trunk - Bug #6392]
+	* test/rdoc/test_rdoc_options.rb:  Test for above.
+	* test/rdoc/test_rdoc_parser.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_c.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_changelog.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_markdown.rb:  ditto.
+	* test/rdoc/test_rdoc_parser_rd.rb:  ditto.
+	* test/rdoc/test_rdoc_rdoc.rb:  ditto.
+
+	* lib/rdoc/tom_doc.rb:  Fixed parsing of [] in TomDoc arguments list.
+	  RDoc bug #167
+	* test/rdoc/test_rdoc_tom_doc.rb:  Test for above.
+
+	* lib/rdoc.rb:  Update version.
+
 Fri Jan  4 11:51:00 2013  Zachary Scott  <zachary@zacharyscott.net>

 	* lib/forwardable.rb: Fix rdoc parameters for ::def_single_delegator.
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@ -64,7 +64,7 @@ module RDoc
  ##
  # RDoc version you are using

-  VERSION = '4.0.0.preview2.1'
+  VERSION = '4.0.0.preview3.1'

  ##
  # Method visibilities
--- a/lib/rdoc/cross_reference.rb
+++ b/lib/rdoc/cross_reference.rb
@ -18,7 +18,7 @@ class RDoc::CrossReference
  #
  # See CLASS_REGEXP_STR

-  METHOD_REGEXP_STR = '([a-z]\w*[!?=]?|%)(?:\([\w.+*/=<>-]*\))?'
+  METHOD_REGEXP_STR = '([a-z]\w*[!?=]?|%|===)(?:\([\w.+*/=<>-]*\))?'

  ##
  # Regular expressions matching text that should potentially have
--- a/lib/rdoc/parser.rb
+++ b/lib/rdoc/parser.rb
@ -58,7 +58,7 @@ class RDoc::Parser
    old_ext = old_ext.sub(/^\.(.*)/, '\1')
    new_ext = new_ext.sub(/^\.(.*)/, '\1')

-    parser = can_parse "xxx.#{old_ext}"
+    parser = can_parse_by_name "xxx.#{old_ext}"
    return false unless parser

    RDoc::Parser.parsers.unshift [/\.#{new_ext}$/, parser]
@ -77,14 +77,14 @@ class RDoc::Parser

    have_encoding = s.respond_to? :encoding

-    if have_encoding then
-      return false if s.encoding != Encoding::ASCII_8BIT and s.valid_encoding?
-    end
-
    return true if s[0, 2] == Marshal.dump('')[0, 2] or s.index("\x00")

    if have_encoding then
-      s.force_encoding Encoding.default_external
+      mode = "r"
+      s.sub!(/\A#!.*\n/, '')     # assume shebang line isn't longer than 1024.
+      encoding = s[/^\s*\#\s*(?:-\*-\s*)?(?:en)?coding:\s*([^\s;]+?)(?:-\*-|[\s;])/, 1]
+      mode = "r:#{encoding}" if encoding
+      s = File.open(file, mode) {|f| f.gets(nil, 1024)}

      not s.valid_encoding?
    else
@ -131,23 +131,36 @@ class RDoc::Parser
    zip_signature == "PK\x03\x04" or
      zip_signature == "PK\x05\x06" or
      zip_signature == "PK\x07\x08"
+  rescue
+    false
  end

  ##
  # Return a parser that can handle a particular extension

-  def self.can_parse(file_name)
-    parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }.last
+  def self.can_parse file_name
+    parser = can_parse_by_name file_name

    # HACK Selenium hides a jar file using a .txt extension
    return if parser == RDoc::Parser::Simple and zip? file_name

+    parser
+  end
+
+  ##
+  # Returns a parser that can handle the extension for +file_name+.  This does
+  # not depend upon the file being readable.
+
+  def self.can_parse_by_name file_name
+    _, parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }
+
    # The default parser must not parse binary files
    ext_name = File.extname file_name
    return parser if ext_name.empty?
+
    if parser == RDoc::Parser::Simple and ext_name !~ /txt|rdoc/ then
      case check_modeline file_name
-      when 'rdoc' then # continue
+      when nil, 'rdoc' then # continue
      else return nil
      end
    end
@ -173,6 +186,8 @@ class RDoc::Parser
      type = $1
    end

+    return nil if /coding:/i =~ type
+
    type.downcase
  rescue ArgumentError # invalid byte sequence, etc.
  end
@ -204,6 +219,8 @@ class RDoc::Parser
    return unless parser

    parser.new top_level, file_name, content, options, stats
+  rescue SystemCallError
+    nil
  end

  ##
--- a/lib/rdoc/parser/changelog.rb
+++ b/lib/rdoc/parser/changelog.rb
@ -102,7 +102,12 @@ class RDoc::Parser::ChangeLog < RDoc::Parser

  def group_entries entries
    entries.group_by do |title, _|
-      Time.parse(title).strftime "%Y-%m-%d"
+      begin
+        Time.parse(title).strftime '%Y-%m-%d'
+      rescue NoMethodError, ArgumentError
+        time, = title.split '  ', 2
+        Time.parse(time).strftime '%Y-%m-%d'
+      end
    end
  end

@ -139,6 +144,9 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
          time = Time.parse entry_name
          # HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other"
          entry_name = nil unless entry_name =~ /#{time.year}/
+        rescue NoMethodError
+          time, = entry_name.split '  ', 2
+          time = Time.parse time
        rescue ArgumentError
          entry_name = nil
        end
--- a/lib/rdoc/tom_doc.rb
+++ b/lib/rdoc/tom_doc.rb
@ -218,7 +218,7 @@ class RDoc::TomDoc < RDoc::Markup::Parser
                   @tokens << [:HEADER, 3, *token_pos(pos)]

                   [:TEXT, @s[1], *token_pos(pos)]
-                 when @s.scan(/([:\w]\w*)[ ]+- /) then
+                 when @s.scan(/([:\w][\w\[\]]*)[ ]+- /) then
                   [:NOTE, @s[1], *token_pos(pos)]
                 else
                   @s.scan(/.*/)
--- a/test/rdoc/test.ja.largedoc
+++ b/test/rdoc/test.ja.largedoc
@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+ 吾輩（わがはい）は猫である。名前はまだ無い。
+ どこで生れたかとんと見当（けんとう）がつかぬ。何でも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している。吾輩はここで始めて人間というものを見た。しかもあとで聞くとそれは書生という人間中で一番獰悪（どうあく）な種族であったそうだ。この書生というのは時々我々を捕（つかま）えて煮（に）て食うという話である。しかしその当時は何という考もなかったから別段恐しいとも思わなかった。ただ彼の掌（てのひら）に載せられてスーと持ち上げられた時何だかフワフワした感じがあったばかりである。掌の上で少し落ちついて書生の顔を見たのがいわゆる人間というものの見始（みはじめ）であろう。この時妙なものだと思った感じが今でも残っている。第一毛をもって装飾されべきはずの顔がつるつるしてまるで薬缶（やかん）だ。その後（ご）猫にもだいぶ逢（あ）ったがこんな片輪（かたわ）には一度も出会（でく）わした事がない。のみならず顔の真中があまりに突起している。そうしてその穴の中から時々ぷうぷうと煙（けむり）を吹く。どうも咽（む）せぽくて実に弱った。これが人間の飲む煙草（たばこ）というものである事はようやくこの頃知った。
--- a/test/rdoc/test_rdoc_cross_reference.rb
+++ b/test/rdoc/test_rdoc_cross_reference.rb
@ -16,6 +16,14 @@ class TestRDocCrossReference < XrefTestCase
    assert_equal name, @xref.resolve(name, name)
  end

+  def test_METHOD_REGEXP_STR
+    re = /#{RDoc::CrossReference::METHOD_REGEXP_STR}/
+
+    re =~ '==='
+
+    assert_equal '===', $&
+  end
+
  def test_resolve_C2
    @xref = RDoc::CrossReference.new @c2

@ -129,6 +137,13 @@ class TestRDocCrossReference < XrefTestCase
    assert_ref @c2_c3_m, '::C2::C3#m(*)'
  end

+  def test_resolve_method_equals3
+    m = RDoc::AnyMethod.new '', '==='
+    @c1.add_method m
+
+    assert_ref m, '==='
+  end
+
  def test_resolve_page
    page = @store.add_file 'README.txt'
    page.parser = RDoc::Parser::Simple
--- a/test/rdoc/test_rdoc_options.rb
+++ b/test/rdoc/test_rdoc_options.rb
@ -549,6 +549,17 @@ rdoc_include:
    FileUtils.rm_rf tmpdir
  end

+  def test_parse_extension_alias
+    out, err = capture_io do
+      @options.parse %w[--extension foobar=rdoc]
+    end
+
+    assert_includes RDoc::Parser.parsers, [/\.foobar$/, RDoc::Parser::Simple]
+
+    assert_empty out
+    assert_empty err
+  end
+
  def test_setup_generator
    test_generator = Class.new do
      def self.setup_options op
--- a/test/rdoc/test_rdoc_parser.rb
+++ b/test/rdoc/test_rdoc_parser.rb
@ -33,8 +33,18 @@ class TestRDocParser < RDoc::TestCase
  end

  def test_class_binary_large_japanese_rdoc
-    file_name = File.expand_path '../test.ja.large.rdoc', __FILE__
-    assert !@RP.binary?(file_name)
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    capture_io do
+      begin
+        extenc, Encoding.default_external =
+          Encoding.default_external, Encoding::US_ASCII
+        file_name = File.expand_path '../test.ja.largedoc', __FILE__
+        assert !@RP.binary?(file_name)
+      ensure
+        Encoding.default_external = extenc
+      end
+    end
  end

  def test_class_binary_japanese_rdoc
@ -51,7 +61,7 @@ class TestRDocParser < RDoc::TestCase

    assert_equal @RP::Simple, @RP.can_parse(readme_file_name)

-    assert_nil @RP.can_parse(@binary_dat)
+    assert_equal @RP::Simple, @RP.can_parse(@binary_dat)

    jtest_file_name = File.expand_path '../test.ja.txt', __FILE__
    assert_equal @RP::Simple, @RP.can_parse(jtest_file_name)
@ -61,20 +71,12 @@ class TestRDocParser < RDoc::TestCase

    readme_file_name = File.expand_path '../README', __FILE__
    assert_equal @RP::Simple, @RP.can_parse(readme_file_name)
-  end

-  def test_class_can_parse_forbidden
-    skip 'chmod not supported' if Gem.win_platform?
+    jtest_largerdoc_file_name = File.expand_path '../test.ja.largedoc', __FILE__
+    assert_equal @RP::Simple, @RP.can_parse(jtest_largerdoc_file_name)

-    Tempfile.open 'forbidden' do |io|
-      begin
-        File.chmod 0000, io.path
-
-        assert_nil @RP.can_parse io.path
-      ensure
-        File.chmod 0400, io.path
-      end
-    end
+    @RP.alias_extension 'rdoc', 'largedoc'
+    assert_equal @RP::Simple, @RP.can_parse(jtest_largerdoc_file_name)
  end

  def test_class_for_executable
@ -82,6 +84,7 @@ class TestRDocParser < RDoc::TestCase
      content = "#!/usr/bin/env ruby -w\n"
      open 'app', 'w' do |io| io.write content end
      app = @store.add_file 'app'
+
      parser = @RP.for app, 'app', content, @options, :stats

      assert_kind_of RDoc::Parser::Ruby, parser
@ -90,6 +93,23 @@ class TestRDocParser < RDoc::TestCase
    end
  end

+  def test_class_for_forbidden
+    skip 'chmod not supported' if Gem.win_platform?
+
+    Tempfile.open 'forbidden' do |io|
+      begin
+        File.chmod 0000, io.path
+        forbidden = @store.add_file io.path
+
+        parser = @RP.for forbidden, 'forbidden', '', @options, :stats
+
+        assert_nil parser
+      ensure
+        File.chmod 0400, io.path
+      end
+    end
+  end
+
  def test_can_parse_modeline
    readme_ext = File.join Dir.tmpdir, "README.EXT.#{$$}"

@ -126,6 +146,18 @@ class TestRDocParser < RDoc::TestCase
    File.unlink readme_ext
  end

+  def test_check_modeline_coding
+    readme_ext = File.join Dir.tmpdir, "README.EXT.#{$$}"
+
+    open readme_ext, 'w' do |io|
+      io.puts "# -*- coding: utf-8 -*-"
+    end
+
+    assert_nil @RP.check_modeline readme_ext
+  ensure
+    File.unlink readme_ext
+  end
+
  def test_check_modeline_with_other
    readme_ext = File.join Dir.tmpdir, "README.EXT.#{$$}"

--- a/test/rdoc/test_rdoc_parser_c.rb
+++ b/test/rdoc/test_rdoc_parser_c.rb
@ -64,17 +64,40 @@ class TestRDocParserC < RDoc::TestCase
  def test_class_can_parse
    c_parser = RDoc::Parser::C

-    assert_equal c_parser, c_parser.can_parse('file.C')
-    assert_equal c_parser, c_parser.can_parse('file.CC')
-    assert_equal c_parser, c_parser.can_parse('file.H')
-    assert_equal c_parser, c_parser.can_parse('file.HH')
-    assert_equal c_parser, c_parser.can_parse('file.c')
-    assert_equal c_parser, c_parser.can_parse('file.cc')
-    assert_equal c_parser, c_parser.can_parse('file.cpp')
-    assert_equal c_parser, c_parser.can_parse('file.cxx')
-    assert_equal c_parser, c_parser.can_parse('file.h')
-    assert_equal c_parser, c_parser.can_parse('file.hh')
-    assert_equal c_parser, c_parser.can_parse('file.y')
+    temp_dir do
+      FileUtils.touch 'file.C'
+      assert_equal c_parser, c_parser.can_parse('file.C')
+
+      FileUtils.touch 'file.CC'
+      assert_equal c_parser, c_parser.can_parse('file.CC')
+
+      FileUtils.touch 'file.H'
+      assert_equal c_parser, c_parser.can_parse('file.H')
+
+      FileUtils.touch 'file.HH'
+      assert_equal c_parser, c_parser.can_parse('file.HH')
+
+      FileUtils.touch 'file.c'
+      assert_equal c_parser, c_parser.can_parse('file.c')
+
+      FileUtils.touch 'file.cc'
+      assert_equal c_parser, c_parser.can_parse('file.cc')
+
+      FileUtils.touch 'file.cpp'
+      assert_equal c_parser, c_parser.can_parse('file.cpp')
+
+      FileUtils.touch 'file.cxx'
+      assert_equal c_parser, c_parser.can_parse('file.cxx')
+
+      FileUtils.touch 'file.h'
+      assert_equal c_parser, c_parser.can_parse('file.h')
+
+      FileUtils.touch 'file.hh'
+      assert_equal c_parser, c_parser.can_parse('file.hh')
+
+      FileUtils.touch 'file.y'
+      assert_equal c_parser, c_parser.can_parse('file.y')
+    end
  end

  def test_initialize
--- a/test/rdoc/test_rdoc_parser_changelog.rb
+++ b/test/rdoc/test_rdoc_parser_changelog.rb
@ -25,11 +25,15 @@ class TestRDocParserChangeLog < RDoc::TestCase
  def test_class_can_parse
    parser = RDoc::Parser::ChangeLog

-    assert_equal parser, parser.can_parse('ChangeLog')
+    temp_dir do
+      FileUtils.touch 'ChangeLog'
+      assert_equal parser, parser.can_parse('ChangeLog')

-    assert_equal parser, parser.can_parse(@tempfile.path)
+      assert_equal parser, parser.can_parse(@tempfile.path)

-    assert_equal RDoc::Parser::Ruby, parser.can_parse('ChangeLog.rb')
+      FileUtils.touch 'ChangeLog.rb'
+      assert_equal RDoc::Parser::Ruby, parser.can_parse('ChangeLog.rb')
+    end
  end

  def test_continue_entry_body
@ -178,7 +182,9 @@ class TestRDocParserChangeLog < RDoc::TestCase
      [ 'Tue Dec  4 08:32:10 2012  Eric Hodel  <drbrain@segment7.net>',
        %w[three four]],
      [ 'Mon Dec  3 20:28:02 2012  Koichi Sasada  <ko1@atdot.net>',
-        %w[five six]]]
+        %w[five six]],
+      [ '2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>',
+        %w[seven eight]]]

    expected = {
      '2012-12-04' => [
@ -189,6 +195,9 @@ class TestRDocParserChangeLog < RDoc::TestCase
      '2012-12-03' => [
        ['Mon Dec  3 20:28:02 2012  Koichi Sasada  <ko1@atdot.net>',
          %w[five six]]],
+      '2008-01-30' => [
+        ['2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>',
+          %w[seven eight]]],
    }

    assert_equal expected, parser.group_entries(entries)
@ -222,6 +231,25 @@ Other note that will be ignored
    assert_equal expected, parser.parse_entries
  end

+  def test_parse_entries_bad_time
+    parser = util_parser <<-ChangeLog
+2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>
+
+        PR libffi/34612
+        * src/x86/sysv.S (ffi_closure_SYSV): Pop 4 byte from stack when
+        returning struct.
+
+    ChangeLog
+
+    expected = [
+      [ '2008-01-30  H.J. Lu  <hongjiu.lu@intel.com>',
+        [ 'src/x86/sysv.S (ffi_closure_SYSV): Pop 4 byte from stack when ' +
+          'returning struct.']]
+    ]
+
+    assert_equal expected, parser.parse_entries
+  end
+
  def test_parse_entries_gnu
    parser = util_parser <<-ChangeLog
 1998-08-17  Richard Stallman  <rms@gnu.org>
--- a/test/rdoc/test_rdoc_parser_markdown.rb
+++ b/test/rdoc/test_rdoc_parser_markdown.rb
@ -27,11 +27,17 @@ class TestRDocParserMarkdown < RDoc::TestCase
  end

  def test_class_can_parse
-    assert_equal @RP::Markdown, @RP.can_parse('foo.md')
-    assert_equal @RP::Markdown, @RP.can_parse('foo.md.ja')
+    temp_dir do
+      FileUtils.touch 'foo.md'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.md')
+      FileUtils.touch 'foo.md.ja'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.md.ja')

-    assert_equal @RP::Markdown, @RP.can_parse('foo.markdown')
-    assert_equal @RP::Markdown, @RP.can_parse('foo.markdown.ja')
+      FileUtils.touch 'foo.markdown'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.markdown')
+      FileUtils.touch 'foo.markdown.ja'
+      assert_equal @RP::Markdown, @RP.can_parse('foo.markdown.ja')
+    end
  end

  def test_scan
--- a/test/rdoc/test_rdoc_parser_rd.rb
+++ b/test/rdoc/test_rdoc_parser_rd.rb
@ -34,8 +34,13 @@ class TestRDocParserRd < RDoc::TestCase
  end

  def test_class_can_parse
-    assert_equal @RP::RD, @RP.can_parse('foo.rd')
-    assert_equal @RP::RD, @RP.can_parse('foo.rd.ja')
+    temp_dir do
+      FileUtils.touch 'foo.rd'
+      assert_equal @RP::RD, @RP.can_parse('foo.rd')
+
+      FileUtils.touch 'foo.rd.ja'
+      assert_equal @RP::RD, @RP.can_parse('foo.rd.ja')
+    end
  end

  def test_scan
--- a/test/rdoc/test_rdoc_rdoc.rb
+++ b/test/rdoc/test_rdoc_rdoc.rb
@ -14,19 +14,22 @@ class TestRDocRDoc < RDoc::TestCase

  def test_document # functional test
    options = RDoc::Options.new
-    options.files = [File.expand_path('../xref_data.rb')]
+    options.files = [File.expand_path('../xref_data.rb', __FILE__)]
    options.setup_generator 'ri'
    options.main_page = 'MAIN_PAGE.rdoc'
+    options.root      = Pathname File.expand_path('..', __FILE__)
    options.title     = 'title'

    rdoc = RDoc::RDoc.new

    temp_dir do
+      options.op_dir = 'ri'
+
      capture_io do
        rdoc.document options
      end

-      assert File.directory? 'doc'
+      assert File.directory? 'ri'
      assert_equal rdoc, rdoc.store.rdoc
    end

--- a/test/rdoc/test_rdoc_tom_doc.rb
+++ b/test/rdoc/test_rdoc_tom_doc.rb
@ -156,6 +156,24 @@ description - arguments description
    assert_equal expected, @TD.parse(text)
  end

+  def test_parse_arguments_array
+    text = <<-TEXT
+Create new Arg object.
+
+names[] - names of arguments
+    TEXT
+
+    expected =
+      doc(
+        para('Create new Arg object.'),
+        blank_line,
+        list(:NOTE,
+          item(%w[names[]],
+            para('names of arguments'))))
+
+    assert_equal expected, @TD.parse(text)
+  end
+
  def test_parse_arguments_multiline
    text = <<-TEXT
 Do some stuff
@ -343,6 +361,25 @@ description - arguments description
    assert_equal expected, @td.tokens
  end

+  def test_tokenize_arguments_array
+    @td.tokenize <<-TEXT
+Create new Arg object.
+
+names[stuff] - names of arguments
+    TEXT
+
+    expected = [
+      [:TEXT,    "Create new Arg object.",  0, 0],
+      [:NEWLINE, "\n",                     22, 0],
+      [:NEWLINE, "\n",                      0, 1],
+      [:NOTE,    "names[stuff]",            0, 2],
+      [:TEXT,    "names of arguments",     15, 2],
+      [:NEWLINE, "\n",                     33, 2],
+    ]
+
+    assert_equal expected, @td.tokens
+  end
+
  def test_tokenize_arguments_multiline
    @td.tokenize <<-TEXT
 Do some stuff