2017-05-02 13:48:10 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
begin
|
2017-07-15 03:24:12 -04:00
|
|
|
require 'ripper'
|
2017-05-02 13:48:10 -04:00
|
|
|
require 'test/unit'
|
|
|
|
ripper_test = true
|
|
|
|
module TestRipper; end
|
|
|
|
rescue LoadError
|
|
|
|
end
|
|
|
|
|
|
|
|
class TestRipper::Lexer < Test::Unit::TestCase
|
|
|
|
def test_nested_dedent_heredoc
|
|
|
|
bug = '[ruby-core:80977] [Bug #13536]'
|
|
|
|
str = <<~'E'
|
|
|
|
<<~"D"
|
|
|
|
#{
|
|
|
|
<<~"B"
|
|
|
|
this must be a valid ruby
|
|
|
|
B
|
|
|
|
}
|
|
|
|
D
|
|
|
|
E
|
|
|
|
assert_equal(str, Ripper.tokenize(str).join(""), bug)
|
2017-05-06 11:14:58 -04:00
|
|
|
|
|
|
|
str = <<~'E'
|
|
|
|
<<~"D"
|
|
|
|
#{
|
|
|
|
<<~"B"
|
|
|
|
this must be a valid ruby
|
|
|
|
B
|
|
|
|
}
|
|
|
|
D
|
|
|
|
E
|
|
|
|
assert_equal(str, Ripper.tokenize(str).join(""), bug)
|
2017-05-02 13:48:10 -04:00
|
|
|
end
|
2017-07-06 00:12:51 -04:00
|
|
|
|
|
|
|
def test_embedded_expr_in_heredoc
|
|
|
|
src = <<~'E'
|
|
|
|
<<~B
|
|
|
|
#{1}
|
|
|
|
B
|
|
|
|
E
|
|
|
|
expect = %I[
|
|
|
|
on_heredoc_beg
|
|
|
|
on_nl
|
|
|
|
on_ignored_sp
|
|
|
|
on_embexpr_beg
|
|
|
|
on_int
|
|
|
|
on_embexpr_end
|
|
|
|
on_tstring_content
|
|
|
|
on_heredoc_end
|
|
|
|
]
|
|
|
|
assert_equal expect, Ripper.lex(src).map {|e| e[1]}
|
|
|
|
end
|
2017-10-19 00:25:30 -04:00
|
|
|
|
|
|
|
def test_space_after_expr_in_heredoc
|
|
|
|
src = <<~'E'
|
|
|
|
<<~B
|
|
|
|
#{1} a
|
|
|
|
B
|
|
|
|
E
|
|
|
|
expect = %I[
|
|
|
|
on_heredoc_beg
|
|
|
|
on_nl
|
|
|
|
on_ignored_sp
|
|
|
|
on_embexpr_beg
|
|
|
|
on_int
|
|
|
|
on_embexpr_end
|
|
|
|
on_tstring_content
|
|
|
|
on_heredoc_end
|
|
|
|
]
|
|
|
|
assert_equal expect, Ripper.lex(src).map {|e| e[1]}
|
|
|
|
end
|
2017-10-19 01:10:38 -04:00
|
|
|
|
|
|
|
def test_expr_at_beginning_in_heredoc
|
|
|
|
src = <<~'E'
|
|
|
|
<<~B
|
|
|
|
a
|
|
|
|
#{1}
|
|
|
|
B
|
|
|
|
E
|
|
|
|
expect = %I[
|
|
|
|
on_heredoc_beg
|
|
|
|
on_nl
|
|
|
|
on_tstring_content
|
|
|
|
on_embexpr_beg
|
|
|
|
on_int
|
|
|
|
on_embexpr_end
|
|
|
|
on_tstring_content
|
|
|
|
on_heredoc_end
|
|
|
|
]
|
|
|
|
assert_equal expect, Ripper.lex(src).map {|e| e[1]}
|
|
|
|
end
|
2018-03-13 09:03:12 -04:00
|
|
|
|
|
|
|
def test_slice
|
|
|
|
assert_equal "string\#{nil}\n",
|
|
|
|
Ripper.slice(%(<<HERE\nstring\#{nil}\nHERE), "heredoc_beg .*? nl $(.*?) heredoc_end", 1)
|
|
|
|
end
|
2019-05-29 08:39:45 -04:00
|
|
|
|
|
|
|
def state(name)
|
|
|
|
Ripper::Lexer::State.new(Ripper.const_get(name))
|
|
|
|
end
|
|
|
|
|
|
|
|
def test_state_after_ivar
|
|
|
|
assert_equal [[1,0],:on_ivar,"@a",state(:EXPR_END)], Ripper.lex("@a").last
|
|
|
|
assert_equal [[1,1],:on_ivar,"@a",state(:EXPR_ENDFN)], Ripper.lex(":@a").last
|
|
|
|
assert_equal [[1,0],:on_ivar,"@1",state(:EXPR_END)], Ripper.lex("@1").last
|
|
|
|
assert_equal [[1,1],:on_ivar,"@1",state(:EXPR_ENDFN)], Ripper.lex(":@1").last
|
|
|
|
end
|
|
|
|
|
|
|
|
def test_state_after_cvar
|
|
|
|
assert_equal [[1,0],:on_cvar,"@@a",state(:EXPR_END)], Ripper.lex("@@a").last
|
|
|
|
assert_equal [[1,1],:on_cvar,"@@a",state(:EXPR_ENDFN)], Ripper.lex(":@@a").last
|
|
|
|
assert_equal [[1,0],:on_cvar,"@@1",state(:EXPR_END)], Ripper.lex("@@1").last
|
|
|
|
assert_equal [[1,1],:on_cvar,"@@1",state(:EXPR_ENDFN)], Ripper.lex(":@@1").last
|
|
|
|
end
|
2019-05-30 08:49:08 -04:00
|
|
|
|
|
|
|
def test_token_aftr_error_heredoc
|
|
|
|
code = "<<A.upcase\n"
|
|
|
|
result = Ripper::Lexer.new(code).scan
|
|
|
|
message = proc {result.pretty_inspect}
|
|
|
|
expected = [
|
|
|
|
[[1, 0], :on_heredoc_beg, "<<A", state(:EXPR_BEG)],
|
2019-05-30 09:03:33 -04:00
|
|
|
[[1, 2], :compile_error, "A", state(:EXPR_BEG), "can't find string \"A\" anywhere before EOF"],
|
2019-05-30 08:49:08 -04:00
|
|
|
[[1, 3], :on_period, ".", state(:EXPR_DOT)],
|
|
|
|
[[1, 4], :on_ident, "upcase", state(:EXPR_ARG)],
|
|
|
|
[[1, 10], :on_nl, "\n", state(:EXPR_BEG)],
|
|
|
|
]
|
|
|
|
pos = 0
|
|
|
|
expected.each_with_index do |ex, i|
|
|
|
|
s = result[i]
|
|
|
|
assert_equal ex, s.to_a, message
|
2019-05-30 09:03:33 -04:00
|
|
|
if pos > s.pos[1]
|
|
|
|
assert_equal pos, s.pos[1] + s.tok.bytesize, message
|
|
|
|
else
|
|
|
|
assert_equal pos, s.pos[1], message
|
|
|
|
pos += s.tok.bytesize
|
|
|
|
end
|
2019-05-30 08:49:08 -04:00
|
|
|
end
|
|
|
|
assert_equal pos, code.bytesize
|
|
|
|
assert_equal expected.size, result.size
|
|
|
|
end
|
2017-05-02 13:48:10 -04:00
|
|
|
end
|