ripper: Ripper::Lexer#scan

* ext/ripper/lib/ripper/lexer.rb (Ripper::Lexer#scan): parses the
  code and returns the result elements including errors.
  [EXPERIMENTAL]

ext/ripper/lib/ripper/lexer.rb

@@ -60,13 +60,13 @@ class Ripper
       def nobits?(i) to_int.nobits?(i) end
     end
 
-    Elem = Struct.new(:pos, :event, :tok, :state) do
+    Elem = Struct.new(:pos, :event, :tok, :state, :message) do
-      def initialize(pos, event, tok, state)
+      def initialize(pos, event, tok, state, message = nil)
-        super(pos, event, tok, State.new(state))
+        super(pos, event, tok, State.new(state), message)
       end
 
       def inspect
-        "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}>"
+        "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
       end
 
       def pretty_print(q)
@@ -76,10 +76,23 @@ class Ripper
           q.breakable
           q.text("token: ")
           tok.pretty_print(q)
+          if message
+            q.breakable
+            q.text("message: ")
+            q.text(message)
+          end
         }
       end
+
+      def to_a
+        a = super
+        a.pop unless a.last
+        a
+      end
     end
 
+    attr_reader :errors
+
     def tokenize
       parse().sort_by(&:pos).map(&:tok)
     end
@@ -88,7 +101,13 @@ class Ripper
       parse().sort_by(&:pos).map(&:to_a)
     end
 
+    # parse the code and returns elements including errors.
+    def scan
+      (parse() + errors).sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
+    end
+
     def parse
+      @errors = []
       @buf = []
       @stack = []
       super
@@ -144,6 +163,12 @@ class Ripper
       @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
     end
 
+    def on_error(mesg)
+      @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+    end
+    alias on_parse_error on_error
+    alias compile_error on_error
+
     (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
       alias_method event, :_push_token
     end

lib/irb/color.rb

@@ -56,7 +56,8 @@ module IRB # :nodoc:
         on_tstring_content: [[RED],                   ALL],
         on_tstring_end:     [[RED],                   ALL],
         on_words_beg:       [[RED],                   ALL],
-        ERROR:              [[RED, REVERSE],          ALL],
+        on_parse_error:     [[RED, REVERSE],          ALL],
+        compile_error:      [[RED, REVERSE],          ALL],
       }
     rescue NameError
       # Give up highlighting Ripper-incompatible older Ruby
@@ -64,17 +65,6 @@ module IRB # :nodoc:
     end
     private_constant :TOKEN_SEQ_EXPRS
 
-    class Lexer < Ripper::Lexer
-      if method_defined?(:token)
-        def on_error(mesg)
-          # :ERROR comes before other :on_ symbols
-          @buf.push Elem.new([lineno(), column()], :ERROR, token(), state())
-        end
-        alias on_parse_error on_error
-        alias compile_error on_error
-      end
-    end
-
     class << self
       def colorable?
         $stdout.tty? && (/mswin|mingw/ =~ RUBY_PLATFORM || (ENV.key?('TERM') && ENV['TERM'] != 'dumb'))
@@ -107,6 +97,10 @@ module IRB # :nodoc:
         "#{seq.map { |s| "\e[#{const_get(s)}m" }.join('')}#{text}#{clear}"
       end
 
+      def scan(code)
+        Ripper::Lexer.new(code).scan
+      end
+
       def colorize_code(code)
         return code unless colorable?
 
@@ -114,7 +108,7 @@ module IRB # :nodoc:
         colored = +''
         length = 0
 
-        Lexer.new(code).parse.sort_by(&:pos).each do |elem|
+        scan(code).each do |elem|
           token = elem.event
           str = elem.tok
           expr = elem.state
@@ -139,7 +133,9 @@ module IRB # :nodoc:
       private
 
       def dispatch_seq(token, expr, str, in_symbol:)
-        if in_symbol
+        if token == :on_parse_error or token == :compile_error
+          TOKEN_SEQ_EXPRS[token][0]
+        elsif in_symbol
           [YELLOW]
         elsif TOKEN_KEYWORDS.fetch(token, []).include?(str)
           [CYAN, BOLD]