* lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when

the heredoc identifier is followed by a line-end.  This allows proper
  display of some HEREDOCs in source view.
* lib/rdoc/ruby_token.rb:  Added TkHEREDOC
* test/rdoc/test_rdoc_ruby_lex.rb:  Test for above.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38443 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,11 @@
+Tue Dec 18 16:31:20 2012  Eric Hodel  <drbrain@segment7.net>
+
+	* lib/rdoc/ruby_lex.rb:  Return a TkHEREDOC instead of a TkSTRING when
+	  the heredoc identifier is followed by a line-end.  This allows proper
+          display of some HEREDOCs in source view.
+	* lib/rdoc/ruby_token.rb:  Added TkHEREDOC
+	* test/rdoc/test_rdoc_ruby_lex.rb:  Test for above.
+
 Tue Dec 18 09:45:14 2012  CHIKANAGA Tomoyuki  <nagachika@ruby-lang.org>
 
 	* vm.c (rb_vm_make_jump_tag_but_local_jump): take care of the case

lib/rdoc/ruby_lex.rb

@@ -982,12 +982,13 @@ class RDoc::RubyLex
       indent = true
     end
     if /['"`]/ =~ ch
-      lt = ch
+      user_quote = lt = ch
       quoted = ""
       while (c = getc) && c != lt
         quoted.concat c
       end
     else
+      user_quote = nil
       lt = '"'
       quoted = ch.dup
       while (c = getc) && c =~ /\w/
@@ -1007,8 +1008,17 @@ class RDoc::RubyLex
       end
     end
 
+    output_heredoc = reserve.join =~ /\A\r?\n\z/
+
+    if output_heredoc then
+      doc = '<<'
+      doc << '-' if indent
+      doc << "#{user_quote}#{quoted}#{user_quote}\n"
+    else
+      doc = '"'
+    end
+
     @here_header = false
-    doc = '"'
     while l = gets
       l = l.sub(/(:?\r)?\n\z/, "\n")
       if (indent ? l.strip : l.chomp) == quoted
@@ -1016,7 +1026,12 @@ class RDoc::RubyLex
       end
       doc << l
     end
-    doc << '"'
+
+    if output_heredoc then
+      doc << l.chomp
+    else
+      doc << '"'
+    end
 
     @here_header = true
     @here_readed.concat reserve
@@ -1024,9 +1039,10 @@ class RDoc::RubyLex
       ungetc ch
     end
 
+    token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
     @ltype = ltback
     @lex_state = EXPR_END
-    Token(Ltype2Token[lt], doc)
+    Token(token_class, doc)
   end
 
   def identify_quotation

lib/rdoc/ruby_token.rb

@@ -331,6 +331,7 @@ module RDoc::RubyToken
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],
     [:TkSTRING,     TkVal],
+    [:TkHEREDOC,    TkVal],
     [:TkXSTRING,    TkVal],
     [:TkREGEXP,     TkVal],
     [:TkSYMBOL,     TkVal],

test/rdoc/test_rdoc_ruby_lex.rb

@@ -70,7 +70,8 @@ end
       @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
       @TK::TkNL        .new( 5, 1,  5, "\n"),
       @TK::TkSPACE     .new( 6, 2,  0, '  '),
-      @TK::TkSTRING    .new( 8, 2,  2, %Q{"Line 1\nLine 2\n"}),
+      @TK::TkHEREDOC   .new( 8, 2,  2,
+                            %Q{<<E\nLine 1\nLine 2\nE}),
       @TK::TkNL        .new(27, 5, 28, "\n"),
       @TK::TkEND       .new(28, 6,  0, 'end'),
       @TK::TkNL        .new(31, 6, 28, "\n"),
@@ -96,12 +97,34 @@ end
     assert_equal expected, tokens
   end
 
-  def test_class_tokenize_heredoc_indent
+  def test_class_tokenize_heredoc_CR_NL
+    tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
+string = <<-STRING\r
+Line 1\r
+Line 2\r
+  STRING\r
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkHEREDOC   .new( 9, 1,  9,
+                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkSPACE     .new(44, 4, 45, "\r"),
+      @TK::TkNL        .new(45, 4, 46, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_call
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING
+string = <<-STRING.chomp
 Line 1
 Line 2
-STRING
+  STRING
     RUBY
 
     expected = [
@@ -110,7 +133,30 @@ STRING
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
       @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
-      @TK::TkNL        .new(39, 4, 40, "\n"),
+      @TK::TkDOT       .new(41, 4, 42, '.'),
+      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
+      @TK::TkNL        .new(47, 4, 48, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_indent
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+string = <<-STRING
+Line 1
+Line 2
+  STRING
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkHEREDOC   .new( 9, 1,  9,
+                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkNL        .new(41, 4, 42, "\n"),
     ]
 
     assert_equal expected, tokens
@@ -128,7 +174,7 @@ U
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
       @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
       @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkSTRING    .new( 4, 1,  4, %Q{"%N\n"}),
+      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
       @TK::TkNL        .new(13, 3, 14, "\n"),
     ]