From cbf6a7f906d17968c8f54f4efb1899e9b204ccfe Mon Sep 17 00:00:00 2001
From: Nobuhiro IMAI <nov@yo.rim.or.jp>
Date: Sat, 21 Nov 2020 08:40:31 +0900
Subject: [PATCH] [ruby/irb] workaround for lack of tokens from `Ripper.lex`

* Fixes #38

https://github.com/ruby/irb/commit/905fb8e52e
---
 lib/irb/ruby-lex.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb
index 51efc17433..03022822bb 100644
--- a/lib/irb/ruby-lex.rb
+++ b/lib/irb/ruby-lex.rb
@@ -88,9 +88,12 @@ class RubyLex
 
   def ripper_lex_without_warning(code)
     verbose, $VERBOSE = $VERBOSE, nil
-    tokens = nil
+    tokens = []
     self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
-      tokens = Ripper.lex(inner_code, '-', line_no)
+      lexer = Ripper::Lexer.new(inner_code, '-', line_no)
+      until (ts = lexer.lex).empty?
+        tokens.concat(ts)
+      end
     end
     $VERBOSE = verbose
     tokens