Always color Symbol as Yellow on IRB::Color

Symbol color was made blue as a workaround because it was hard to
distinguish `foo`s in `:foo` and `def foo; end` (both are :on_ident).
But I wanted to make it yellow like pry.

`:Struct` had the same problem in :on_const. Because the :on_const was
also blue (but underlined and bold), it was not a big issue.

While they're not so problematic since we got a workaround, we also had
a more serious issue for highlighting a symbol like `:"a#{b}c"`.
The first half was considered as Symbol and the last half was considered
as String, because the colorizer did not have a state like a parser.

To approach the last issue, I introduced `IRB::Color::SymbolState` which
is a thin state manager knowing only "the token is Symbol or not". Having
this module magically solves the first two problems as well. So now we
can highlight Symbol as yellow in the perfect manner.

lib/irb/color.rb

@@ -9,6 +9,7 @@ module IRB # :nodoc:
     UNDERLINE = 4
     RED       = 31
     GREEN     = 32
+    YELLOW    = 33
     BLUE      = 34
     MAGENTA   = 35
     CYAN      = 36
@@ -37,13 +38,22 @@ module IRB # :nodoc:
         on_qsymbols_beg:    [[RED],                   [Ripper::EXPR_BEG]],
         on_regexp_beg:      [[RED, BOLD],             [Ripper::EXPR_BEG]],
         on_regexp_end:      [[RED, BOLD],             [Ripper::EXPR_BEG]],
-        on_symbeg:          [[BLUE, BOLD],            [Ripper::EXPR_FNAME]],
+        on_symbeg:          [[YELLOW],                [Ripper::EXPR_FNAME]],
         on_tstring_beg:     [[RED],                   [Ripper::EXPR_BEG, Ripper::EXPR_END, Ripper::EXPR_ARG, Ripper::EXPR_CMDARG]],
         on_tstring_content: [[RED],                   [Ripper::EXPR_BEG, Ripper::EXPR_END, Ripper::EXPR_ARG, Ripper::EXPR_CMDARG, Ripper::EXPR_FNAME]],
         on_tstring_end:     [[RED],                   [Ripper::EXPR_END]],
       }
+      SYMBOL_SEQ_OVERRIDES = {
+        on_const:           [YELLOW],
+        on_embexpr_beg:     [YELLOW],
+        on_embexpr_end:     [YELLOW],
+        on_ident:           [YELLOW],
+        on_tstring_content: [YELLOW],
+        on_tstring_end:     [YELLOW],
+      }
     rescue NameError
       TOKEN_SEQ_EXPRS = {}
+      SYMBOL_SEQ_OVERRIDES = {}
     end
 
     class << self
@@ -81,10 +91,13 @@ module IRB # :nodoc:
       def colorize_code(code)
         return code unless colorable?
 
+        symbol_state = SymbolState.new
         colored = +''
         length = 0
+
         Ripper.lex(code).each do |(_line, _col), token, str, expr|
-          if seq = dispatch_seq(token, expr, str)
+          in_symbol = symbol_state.scan_token(token)
+          if seq = dispatch_seq(token, expr, str, in_symbol: in_symbol)
             Reline::Unicode.escape_for_print(str).each_line do |line|
               colored << "#{seq.map { |s| "\e[#{s}m" }.join('')}#{line.sub(/\n?\z/, "#{clear}\\0")}"
             end
@@ -102,17 +115,51 @@ module IRB # :nodoc:
 
       private
 
-      def dispatch_seq(token, expr, str)
+      def dispatch_seq(token, expr, str, in_symbol:)
         if token == :on_comment
           [BLUE, BOLD]
         elsif TOKEN_KEYWORDS.fetch(token, []).include?(str)
           [CYAN, BOLD]
         elsif (seq, exprs = TOKEN_SEQ_EXPRS[token]; exprs&.any? { |e| (expr & e) != 0 })
-          seq
+          SYMBOL_SEQ_OVERRIDES.fetch(in_symbol ? token : nil, seq)
         else
           nil
         end
       end
     end
+
+    # A class to manage a state to know whether the current token is for Symbol or not.
+    class SymbolState
+      def initialize
+        # Push `true` to detect Symbol. `false` to increase the nest level for non-Symbol.
+        @stack = []
+      end
+
+      # Return true if the token is a part of Symbol.
+      def scan_token(token)
+        prev_state = @stack.last
+        case token
+        when :on_symbeg
+          @stack << true
+        when :on_ident
+          if @stack.last # Pop only when it's :sym
+            @stack.pop
+            return prev_state
+          end
+        when :on_tstring_beg
+          @stack << false
+        when :on_embexpr_beg
+          @stack << false
+          return prev_state
+        when :on_tstring_end # :on_tstring_end may close Symbol
+          @stack.pop
+          return prev_state
+        when :on_embexpr_end
+          @stack.pop
+        end
+        @stack.last
+      end
+    end
+    private_constant :SymbolState
   end
 end

test/irb/test_color.rb

@@ -11,6 +11,7 @@ module TestIRB
     UNDERLINE = "\e[4m"
     RED       = "\e[31m"
     GREEN     = "\e[32m"
+    YELLOW    = "\e[33m"
     BLUE      = "\e[34m"
     MAGENTA   = "\e[35m"
     CYAN      = "\e[36m"
@@ -24,7 +25,7 @@ module TestIRB
       {
         "1" => "#{BLUE}#{BOLD}1#{CLEAR}",
         "2.3" => "#{MAGENTA}#{BOLD}2.3#{CLEAR}",
-        "['foo', :bar]" => "[#{RED}'#{CLEAR}#{RED}foo#{CLEAR}#{RED}'#{CLEAR}, #{BLUE}#{BOLD}:#{CLEAR}#{BLUE}#{BOLD}bar#{CLEAR}]",
+        "['foo', :bar]" => "[#{RED}'#{CLEAR}#{RED}foo#{CLEAR}#{RED}'#{CLEAR}, #{YELLOW}:#{CLEAR}#{YELLOW}bar#{CLEAR}]",
         "class A; end" => "#{GREEN}class#{CLEAR} #{BLUE}#{BOLD}#{UNDERLINE}A#{CLEAR}; #{GREEN}end#{CLEAR}",
         "def self.foo; bar; end" => "#{GREEN}def#{CLEAR} #{CYAN}#{BOLD}self#{CLEAR}.#{BLUE}#{BOLD}foo#{CLEAR}; bar; #{GREEN}end#{CLEAR}",
         'ERB.new("a#{nil}b", trim_mode: "-")' => "#{BLUE}#{BOLD}#{UNDERLINE}ERB#{CLEAR}.new(#{RED}\"#{CLEAR}#{RED}a#{CLEAR}#{RED}\#{#{CLEAR}#{CYAN}#{BOLD}nil#{CLEAR}#{RED}}#{CLEAR}#{RED}b#{CLEAR}#{RED}\"#{CLEAR}, #{MAGENTA}trim_mode:#{CLEAR} #{RED}\"#{CLEAR}#{RED}-#{CLEAR}#{RED}\"#{CLEAR})",
@@ -40,9 +41,11 @@ module TestIRB
         "%w[a b]" => "#{RED}%w[#{CLEAR}#{RED}a#{CLEAR} #{RED}b#{CLEAR}#{RED}]#{CLEAR}",
         "%i[c d]" => "#{RED}%i[#{CLEAR}#{RED}c#{CLEAR} #{RED}d#{CLEAR}#{RED}]#{CLEAR}",
         "{'a': 1}" => "{#{RED}'#{CLEAR}#{RED}a#{CLEAR}#{RED}':#{CLEAR} #{BLUE}#{BOLD}1#{CLEAR}}",
-        ":Struct" => "#{BLUE}#{BOLD}:#{CLEAR}#{BLUE}#{BOLD}#{UNDERLINE}Struct#{CLEAR}",
+        ":Struct" => "#{YELLOW}:#{CLEAR}#{YELLOW}Struct#{CLEAR}",
         "<<EOS\nhere\nEOS" => "#{RED}<<EOS#{CLEAR}\n#{RED}here#{CLEAR}\n#{RED}EOS#{CLEAR}",
         '"#{}"' => "#{RED}\"#{CLEAR}#{RED}\#{#{CLEAR}#{RED}}#{CLEAR}#{RED}\"#{CLEAR}",
+        ':"a#{}b"' => "#{YELLOW}:\"#{CLEAR}#{YELLOW}a#{CLEAR}#{YELLOW}\#{#{CLEAR}#{YELLOW}}#{CLEAR}#{YELLOW}b#{CLEAR}#{YELLOW}\"#{CLEAR}",
+        ':"a#{ def b; end; \'c\' + "#{ :d }" }e"' => "#{YELLOW}:\"#{CLEAR}#{YELLOW}a#{CLEAR}#{YELLOW}\#{#{CLEAR} #{GREEN}def#{CLEAR} #{BLUE}#{BOLD}b#{CLEAR}; #{GREEN}end#{CLEAR}; #{RED}'#{CLEAR}#{RED}c#{CLEAR}#{RED}'#{CLEAR} + #{RED}\"#{CLEAR}#{RED}\#{#{CLEAR} #{YELLOW}:#{CLEAR}#{YELLOW}d#{CLEAR} #{RED}}#{CLEAR}#{RED}\"#{CLEAR} #{YELLOW}}#{CLEAR}#{YELLOW}e#{CLEAR}#{YELLOW}\"#{CLEAR}",
       }.each do |code, result|
         actual = with_term { IRB::Color.colorize_code(code) }
         assert_equal(result, actual, "Case: colorize_code(#{code.dump})\nResult: #{humanized_literal(actual)}")
@@ -91,6 +94,7 @@ module TestIRB
         .gsub(UNDERLINE, '@@@{UNDERLINE}')
         .gsub(RED, '@@@{RED}')
         .gsub(GREEN, '@@@{GREEN}')
+        .gsub(YELLOW, '@@@{YELLOW}')
         .gsub(BLUE, '@@@{BLUE}')
         .gsub(MAGENTA, '@@@{MAGENTA}')
         .gsub(CYAN, '@@@{CYAN}')