ripper: fix a bug of Ripper::Lexer with syntax error and heredoc [Bug #17644]

ext/ripper/lib/ripper/lexer.rb

@@ -136,7 +136,7 @@ class Ripper
       end
       @buf.flatten!
       unless (result = @buf).empty?
-        result.concat(@buf) until (@buf = []; super(); @buf.empty?)
+        result.concat(@buf) until (@buf = []; super(); @buf.flatten!; @buf.empty?)
       end
       result
     end

test/ripper/test_lexer.rb

@@ -216,4 +216,16 @@ class TestRipper::Lexer < Test::Unit::TestCase
       end
     end
   end
+
+  def test_lex_with_syntax_error_and_heredo
+    bug = '[Bug #17644]'
+    s = <<~EOF
+        foo
+      end
+      <<~EOS
+        bar
+      EOS
+    EOF
+    assert_equal([[5, 0], :on_heredoc_end, "EOS\n", state(:EXPR_BEG)], Ripper.lex(s).last, bug)
+  end
 end