mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
80325bab52
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@47521 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
1377 lines
28 KiB
Ruby
1377 lines
28 KiB
Ruby
# coding: US-ASCII
|
|
|
|
#--
|
|
# irb/ruby-lex.rb - ruby lexcal analyzer
|
|
# $Release Version: 0.9.5$
|
|
# $Revision: 17979 $
|
|
# $Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $
|
|
# by Keiju ISHITSUKA(keiju@ruby-lang.org)
|
|
#
|
|
#++
|
|
|
|
require "e2mmap"
|
|
require "irb/slex"
|
|
require "stringio"
|
|
|
|
##
|
|
# Ruby lexer adapted from irb.
|
|
#
|
|
# The internals are not documented because they are scary.
|
|
|
|
class RDoc::RubyLex
|
|
|
|
##
|
|
# Raised upon invalid input
|
|
|
|
class Error < RDoc::Error
|
|
end
|
|
|
|
# :stopdoc:
|
|
|
|
extend Exception2MessageMapper
|
|
|
|
def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
|
|
def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
|
|
def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
|
|
def_exception(:TkReading2TokenDuplicateError,
|
|
"key duplicate(token_n='%s', key='%s')")
|
|
def_exception(:SyntaxError, "%s")
|
|
|
|
def_exception(:TerminateLineInput, "Terminate Line Input")
|
|
|
|
include RDoc::RubyToken
|
|
include IRB
|
|
|
|
attr_accessor :continue
|
|
attr_accessor :lex_state
|
|
attr_reader :reader
|
|
|
|
class << self
|
|
attr_accessor :debug_level
|
|
end
|
|
|
|
def self.debug?
|
|
@debug_level > 0
|
|
end
|
|
|
|
self.debug_level = 0
|
|
|
|
# :startdoc:
|
|
|
|
##
|
|
# Returns an Array of +ruby+ tokens. See ::new for a description of
|
|
# +options+.
|
|
|
|
def self.tokenize ruby, options
|
|
tokens = []
|
|
|
|
scanner = RDoc::RubyLex.new ruby, options
|
|
scanner.exception_on_syntax_error = true
|
|
|
|
while token = scanner.token do
|
|
tokens << token
|
|
end
|
|
|
|
tokens
|
|
end
|
|
|
|
##
|
|
# Creates a new lexer for +content+. +options+ is an RDoc::Options, only
|
|
# +tab_width is used.
|
|
|
|
def initialize(content, options)
|
|
lex_init
|
|
|
|
if /\t/ =~ content then
|
|
tab_width = options.tab_width
|
|
content = content.split(/\n/).map do |line|
|
|
1 while line.gsub!(/\t+/) {
|
|
' ' * (tab_width*$&.length - $`.length % tab_width)
|
|
} && $~
|
|
line
|
|
end.join("\n")
|
|
end
|
|
|
|
content << "\n" unless content[-1, 1] == "\n"
|
|
|
|
set_input StringIO.new content
|
|
|
|
@base_char_no = 0
|
|
@char_no = 0
|
|
@exp_line_no = @line_no = 1
|
|
@here_readed = []
|
|
@readed = []
|
|
@rests = []
|
|
@seek = 0
|
|
|
|
@here_header = false
|
|
@indent = 0
|
|
@indent_stack = []
|
|
@lex_state = :EXPR_BEG
|
|
@space_seen = false
|
|
|
|
@continue = false
|
|
@line = ""
|
|
|
|
@skip_space = false
|
|
@readed_auto_clean_up = false
|
|
@exception_on_syntax_error = true
|
|
|
|
@prompt = nil
|
|
@prev_seek = nil
|
|
@ltype = nil
|
|
end
|
|
|
|
# :stopdoc:
|
|
|
|
def inspect # :nodoc:
|
|
"#<%s:0x%x pos %d lex_state %p space_seen %p>" % [
|
|
self.class, object_id,
|
|
@io.pos, @lex_state, @space_seen,
|
|
]
|
|
end
|
|
|
|
attr_accessor :skip_space
|
|
attr_accessor :readed_auto_clean_up
|
|
attr_accessor :exception_on_syntax_error
|
|
|
|
attr_reader :seek
|
|
attr_reader :char_no
|
|
attr_reader :line_no
|
|
attr_reader :indent
|
|
|
|
# io functions
|
|
def set_input(io, p = nil, &block)
|
|
@io = io
|
|
if p.respond_to?(:call)
|
|
@input = p
|
|
elsif block_given?
|
|
@input = block
|
|
else
|
|
@input = Proc.new{@io.gets}
|
|
end
|
|
end
|
|
|
|
def get_readed
|
|
if idx = @readed.rindex("\n")
|
|
@base_char_no = @readed.size - (idx + 1)
|
|
else
|
|
@base_char_no += @readed.size
|
|
end
|
|
|
|
readed = @readed.join("")
|
|
@readed = []
|
|
readed
|
|
end
|
|
|
|
def getc
|
|
while @rests.empty?
|
|
# return nil unless buf_input
|
|
@rests.push nil unless buf_input
|
|
end
|
|
c = @rests.shift
|
|
if @here_header
|
|
@here_readed.push c
|
|
else
|
|
@readed.push c
|
|
end
|
|
@seek += 1
|
|
if c == "\n"
|
|
@line_no += 1
|
|
@char_no = 0
|
|
else
|
|
@char_no += 1
|
|
end
|
|
|
|
c
|
|
end
|
|
|
|
def gets
|
|
l = ""
|
|
while c = getc
|
|
l.concat(c)
|
|
break if c == "\n"
|
|
end
|
|
return nil if l == "" and c.nil?
|
|
l
|
|
end
|
|
|
|
def eof?
|
|
@io.eof?
|
|
end
|
|
|
|
def getc_of_rests
|
|
if @rests.empty?
|
|
nil
|
|
else
|
|
getc
|
|
end
|
|
end
|
|
|
|
def ungetc(c = nil)
|
|
if @here_readed.empty?
|
|
c2 = @readed.pop
|
|
else
|
|
c2 = @here_readed.pop
|
|
end
|
|
c = c2 unless c
|
|
@rests.unshift c #c =
|
|
@seek -= 1
|
|
if c == "\n"
|
|
@line_no -= 1
|
|
if idx = @readed.rindex("\n")
|
|
@char_no = idx + 1
|
|
else
|
|
@char_no = @base_char_no + @readed.size
|
|
end
|
|
else
|
|
@char_no -= 1
|
|
end
|
|
end
|
|
|
|
def peek_equal?(str)
|
|
chrs = str.split(//)
|
|
until @rests.size >= chrs.size
|
|
return false unless buf_input
|
|
end
|
|
@rests[0, chrs.size] == chrs
|
|
end
|
|
|
|
def peek_match?(regexp)
|
|
while @rests.empty?
|
|
return false unless buf_input
|
|
end
|
|
regexp =~ @rests.join("")
|
|
end
|
|
|
|
def peek(i = 0)
|
|
while @rests.size <= i
|
|
return nil unless buf_input
|
|
end
|
|
@rests[i]
|
|
end
|
|
|
|
def buf_input
|
|
prompt
|
|
line = @input.call
|
|
return nil unless line
|
|
@rests.concat line.split(//)
|
|
true
|
|
end
|
|
private :buf_input
|
|
|
|
def set_prompt(p = nil, &block)
|
|
p = block if block_given?
|
|
if p.respond_to?(:call)
|
|
@prompt = p
|
|
else
|
|
@prompt = Proc.new{print p}
|
|
end
|
|
end
|
|
|
|
def prompt
|
|
if @prompt
|
|
@prompt.call(@ltype, @indent, @continue, @line_no)
|
|
end
|
|
end
|
|
|
|
def initialize_input
|
|
@ltype = nil
|
|
@quoted = nil
|
|
@indent = 0
|
|
@indent_stack = []
|
|
@lex_state = :EXPR_BEG
|
|
@space_seen = false
|
|
@here_header = false
|
|
|
|
@continue = false
|
|
prompt
|
|
|
|
@line = ""
|
|
@exp_line_no = @line_no
|
|
end
|
|
|
|
def each_top_level_statement
|
|
initialize_input
|
|
catch(:TERM_INPUT) do
|
|
loop do
|
|
begin
|
|
@continue = false
|
|
prompt
|
|
unless l = lex
|
|
throw :TERM_INPUT if @line == ''
|
|
else
|
|
#p l
|
|
@line.concat l
|
|
if @ltype or @continue or @indent > 0
|
|
next
|
|
end
|
|
end
|
|
if @line != "\n"
|
|
yield @line, @exp_line_no
|
|
end
|
|
break unless l
|
|
@line = ''
|
|
@exp_line_no = @line_no
|
|
|
|
@indent = 0
|
|
@indent_stack = []
|
|
prompt
|
|
rescue TerminateLineInput
|
|
initialize_input
|
|
prompt
|
|
get_readed
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def lex
|
|
until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
|
|
!@continue or
|
|
tk.nil?)
|
|
#p tk
|
|
#p @lex_state
|
|
#p self
|
|
end
|
|
line = get_readed
|
|
# print self.inspect
|
|
if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
|
|
nil
|
|
else
|
|
line
|
|
end
|
|
end
|
|
|
|
def token
|
|
# require "tracer"
|
|
# Tracer.on
|
|
@prev_seek = @seek
|
|
@prev_line_no = @line_no
|
|
@prev_char_no = @char_no
|
|
begin
|
|
begin
|
|
tk = @OP.match(self)
|
|
@space_seen = tk.kind_of?(TkSPACE)
|
|
rescue SyntaxError => e
|
|
raise Error, "syntax error: #{e.message}" if
|
|
@exception_on_syntax_error
|
|
|
|
tk = TkError.new(@seek, @line_no, @char_no)
|
|
end
|
|
end while @skip_space and tk.kind_of?(TkSPACE)
|
|
|
|
if @readed_auto_clean_up
|
|
get_readed
|
|
end
|
|
# Tracer.off
|
|
tk
|
|
end
|
|
|
|
ENINDENT_CLAUSE = [
|
|
"case", "class", "def", "do", "for", "if",
|
|
"module", "unless", "until", "while", "begin" #, "when"
|
|
]
|
|
|
|
DEINDENT_CLAUSE = ["end" #, "when"
|
|
]
|
|
|
|
PERCENT_LTYPE = {
|
|
"q" => "\'",
|
|
"Q" => "\"",
|
|
"x" => "\`",
|
|
"r" => "/",
|
|
"w" => "]",
|
|
"W" => "]",
|
|
"s" => ":"
|
|
}
|
|
|
|
PERCENT_PAREN = {
|
|
"{" => "}",
|
|
"[" => "]",
|
|
"<" => ">",
|
|
"(" => ")"
|
|
}
|
|
|
|
PERCENT_PAREN_REV = PERCENT_PAREN.invert
|
|
|
|
Ltype2Token = {
|
|
"\'" => TkSTRING,
|
|
"\"" => TkSTRING,
|
|
"\`" => TkXSTRING,
|
|
"/" => TkREGEXP,
|
|
"]" => TkDSTRING,
|
|
":" => TkSYMBOL
|
|
}
|
|
DLtype2Token = {
|
|
"\"" => TkDSTRING,
|
|
"\`" => TkDXSTRING,
|
|
"/" => TkDREGEXP,
|
|
}
|
|
|
|
def lex_init()
|
|
@OP = IRB::SLex.new
|
|
@OP.def_rules("\0", "\004", "\032") do |op, io|
|
|
Token(TkEND_OF_SCRIPT, '')
|
|
end
|
|
|
|
@OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
|
|
@space_seen = true
|
|
str = op
|
|
while (ch = getc) =~ /[ \t\f\r\13]/ do
|
|
str << ch
|
|
end
|
|
ungetc
|
|
Token TkSPACE, str
|
|
end
|
|
|
|
@OP.def_rule("#") do |op, io|
|
|
identify_comment
|
|
end
|
|
|
|
@OP.def_rule("=begin",
|
|
proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
|
|
|op, io|
|
|
@ltype = "="
|
|
res = ''
|
|
nil until getc == "\n"
|
|
|
|
until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
|
|
(ch = getc)
|
|
res << ch
|
|
end
|
|
|
|
gets # consume =end
|
|
|
|
@ltype = nil
|
|
Token(TkRD_COMMENT, res)
|
|
end
|
|
|
|
@OP.def_rule("\n") do |op, io|
|
|
print "\\n\n" if RDoc::RubyLex.debug?
|
|
case @lex_state
|
|
when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
|
|
@continue = true
|
|
else
|
|
@continue = false
|
|
@lex_state = :EXPR_BEG
|
|
until (@indent_stack.empty? ||
|
|
[TkLPAREN, TkLBRACK, TkLBRACE,
|
|
TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
|
|
@indent_stack.pop
|
|
end
|
|
end
|
|
@here_header = false
|
|
@here_readed = []
|
|
Token(TkNL)
|
|
end
|
|
|
|
@OP.def_rules("*", "**",
|
|
"=", "==", "===",
|
|
"=~", "<=>",
|
|
"<", "<=",
|
|
">", ">=", ">>") do
|
|
|op, io|
|
|
case @lex_state
|
|
when :EXPR_FNAME, :EXPR_DOT
|
|
@lex_state = :EXPR_ARG
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
end
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rules("!", "!=", "!~") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rules("<<") do
|
|
|op, io|
|
|
tk = nil
|
|
if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
|
|
(@lex_state != :EXPR_ARG || @space_seen)
|
|
c = peek(0)
|
|
if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-")
|
|
tk = identify_here_document
|
|
end
|
|
end
|
|
unless tk
|
|
tk = Token(op)
|
|
case @lex_state
|
|
when :EXPR_FNAME, :EXPR_DOT
|
|
@lex_state = :EXPR_ARG
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
end
|
|
end
|
|
tk
|
|
end
|
|
|
|
@OP.def_rules("'", '"') do
|
|
|op, io|
|
|
identify_string(op)
|
|
end
|
|
|
|
@OP.def_rules("`") do
|
|
|op, io|
|
|
if @lex_state == :EXPR_FNAME
|
|
@lex_state = :EXPR_END
|
|
Token(op)
|
|
else
|
|
identify_string(op)
|
|
end
|
|
end
|
|
|
|
@OP.def_rules('?') do
|
|
|op, io|
|
|
if @lex_state == :EXPR_END
|
|
@lex_state = :EXPR_BEG
|
|
Token(TkQUESTION)
|
|
else
|
|
ch = getc
|
|
if @lex_state == :EXPR_ARG && ch =~ /\s/
|
|
ungetc
|
|
@lex_state = :EXPR_BEG;
|
|
Token(TkQUESTION)
|
|
else
|
|
@lex_state = :EXPR_END
|
|
Token(TkCHAR, "?#{ch}")
|
|
end
|
|
end
|
|
end
|
|
|
|
@OP.def_rules("&", "&&", "|", "||") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rules("+=", "-=", "*=", "**=",
|
|
"&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
op =~ /^(.*)=$/
|
|
Token(TkOPASGN, $1)
|
|
end
|
|
|
|
@OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|
|
|op, io|
|
|
@lex_state = :EXPR_ARG
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|
|
|op, io|
|
|
@lex_state = :EXPR_ARG
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rules("+", "-") do
|
|
|op, io|
|
|
catch(:RET) do
|
|
if @lex_state == :EXPR_ARG
|
|
if @space_seen and peek(0) =~ /[0-9]/
|
|
throw :RET, identify_number(op)
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
end
|
|
elsif @lex_state != :EXPR_END and peek(0) =~ /[0-9]/
|
|
throw :RET, identify_number(op)
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
end
|
|
Token(op)
|
|
end
|
|
end
|
|
|
|
@OP.def_rule(".") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
if peek(0) =~ /[0-9]/
|
|
ungetc
|
|
identify_number
|
|
else
|
|
# for "obj.if" etc.
|
|
@lex_state = :EXPR_DOT
|
|
Token(TkDOT)
|
|
end
|
|
end
|
|
|
|
@OP.def_rules("..", "...") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token(op)
|
|
end
|
|
|
|
lex_int2
|
|
end
|
|
|
|
def lex_int2
|
|
@OP.def_rules("]", "}", ")") do
|
|
|op, io|
|
|
@lex_state = :EXPR_END
|
|
@indent -= 1
|
|
@indent_stack.pop
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rule(":") do
|
|
|op, io|
|
|
if @lex_state == :EXPR_END || peek(0) =~ /\s/
|
|
@lex_state = :EXPR_BEG
|
|
Token(TkCOLON)
|
|
else
|
|
@lex_state = :EXPR_FNAME;
|
|
Token(TkSYMBEG)
|
|
end
|
|
end
|
|
|
|
@OP.def_rule("::") do
|
|
|op, io|
|
|
# p @lex_state.id2name, @space_seen
|
|
if @lex_state == :EXPR_BEG or @lex_state == :EXPR_ARG && @space_seen
|
|
@lex_state = :EXPR_BEG
|
|
Token(TkCOLON3)
|
|
else
|
|
@lex_state = :EXPR_DOT
|
|
Token(TkCOLON2)
|
|
end
|
|
end
|
|
|
|
@OP.def_rule("/") do
|
|
|op, io|
|
|
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
|
|
identify_string(op)
|
|
elsif peek(0) == '='
|
|
getc
|
|
@lex_state = :EXPR_BEG
|
|
Token(TkOPASGN, "/") #/)
|
|
elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
|
|
identify_string(op)
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
Token("/") #/)
|
|
end
|
|
end
|
|
|
|
@OP.def_rules("^") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token("^")
|
|
end
|
|
|
|
# @OP.def_rules("^=") do
|
|
# @lex_state = :EXPR_BEG
|
|
# Token(OP_ASGN, :^)
|
|
# end
|
|
|
|
@OP.def_rules(",") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rules(";") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
until (@indent_stack.empty? ||
|
|
[TkLPAREN, TkLBRACK, TkLBRACE,
|
|
TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
|
|
@indent_stack.pop
|
|
end
|
|
Token(op)
|
|
end
|
|
|
|
@OP.def_rule("~") do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token("~")
|
|
end
|
|
|
|
@OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|
|
|op, io|
|
|
@lex_state = :EXPR_BEG
|
|
Token("~")
|
|
end
|
|
|
|
@OP.def_rule("(") do
|
|
|op, io|
|
|
@indent += 1
|
|
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
|
|
@lex_state = :EXPR_BEG
|
|
tk_c = TkfLPAREN
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
tk_c = TkLPAREN
|
|
end
|
|
@indent_stack.push tk_c
|
|
Token tk_c
|
|
end
|
|
|
|
@OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|
|
|op, io|
|
|
@lex_state = :EXPR_ARG
|
|
Token("[]")
|
|
end
|
|
|
|
@OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|
|
|op, io|
|
|
@lex_state = :EXPR_ARG
|
|
Token("[]=")
|
|
end
|
|
|
|
@OP.def_rule("[") do
|
|
|op, io|
|
|
@indent += 1
|
|
if @lex_state == :EXPR_FNAME
|
|
tk_c = TkfLBRACK
|
|
else
|
|
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
|
|
tk_c = TkLBRACK
|
|
elsif @lex_state == :EXPR_ARG && @space_seen
|
|
tk_c = TkLBRACK
|
|
else
|
|
tk_c = TkfLBRACK
|
|
end
|
|
@lex_state = :EXPR_BEG
|
|
end
|
|
@indent_stack.push tk_c
|
|
Token(tk_c)
|
|
end
|
|
|
|
@OP.def_rule("{") do
|
|
|op, io|
|
|
@indent += 1
|
|
if @lex_state != :EXPR_END && @lex_state != :EXPR_ARG
|
|
tk_c = TkLBRACE
|
|
else
|
|
tk_c = TkfLBRACE
|
|
end
|
|
@lex_state = :EXPR_BEG
|
|
@indent_stack.push tk_c
|
|
Token(tk_c)
|
|
end
|
|
|
|
@OP.def_rule('\\') do
|
|
|op, io|
|
|
if getc == "\n"
|
|
@space_seen = true
|
|
@continue = true
|
|
Token(TkSPACE)
|
|
else
|
|
ungetc
|
|
Token("\\")
|
|
end
|
|
end
|
|
|
|
@OP.def_rule('%') do
|
|
|op, io|
|
|
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
|
|
identify_quotation
|
|
elsif peek(0) == '='
|
|
getc
|
|
Token(TkOPASGN, :%)
|
|
elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
|
|
identify_quotation
|
|
else
|
|
@lex_state = :EXPR_BEG
|
|
Token("%") #))
|
|
end
|
|
end
|
|
|
|
@OP.def_rule('$') do
|
|
|op, io|
|
|
identify_gvar
|
|
end
|
|
|
|
@OP.def_rule('@') do
|
|
|op, io|
|
|
if peek(0) =~ /[\w@]/
|
|
ungetc
|
|
identify_identifier
|
|
else
|
|
Token("@")
|
|
end
|
|
end
|
|
|
|
# @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
|
|
# |op, io|
|
|
# @indent += 1
|
|
# @lex_state = :EXPR_FNAME
|
|
# # @lex_state = :EXPR_END
|
|
# # until @rests[0] == "\n" or @rests[0] == ";"
|
|
# # rests.shift
|
|
# # end
|
|
# end
|
|
|
|
@OP.def_rule("_") do
|
|
if peek_match?(/_END__/) and @lex_state == :EXPR_BEG then
|
|
6.times { getc }
|
|
Token(TkEND_OF_SCRIPT, '__END__')
|
|
else
|
|
ungetc
|
|
identify_identifier
|
|
end
|
|
end
|
|
|
|
@OP.def_rule("") do
|
|
|op, io|
|
|
printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
|
|
if peek(0) =~ /[0-9]/
|
|
t = identify_number
|
|
else
|
|
t = identify_identifier
|
|
end
|
|
printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
|
|
t
|
|
end
|
|
|
|
p @OP if RDoc::RubyLex.debug?
|
|
end
|
|
|
|
def identify_gvar
|
|
@lex_state = :EXPR_END
|
|
|
|
case ch = getc
|
|
when /[~_*$?!@\/\\;,=:<>".]/ #"
|
|
Token(TkGVAR, "$" + ch)
|
|
when "-"
|
|
Token(TkGVAR, "$-" + getc)
|
|
when "&", "`", "'", "+"
|
|
Token(TkBACK_REF, "$"+ch)
|
|
when /[1-9]/
|
|
ref = ch
|
|
while (ch = getc) =~ /[0-9]/ do ref << ch end
|
|
ungetc
|
|
Token(TkNTH_REF, "$#{ref}")
|
|
when /\w/
|
|
ungetc
|
|
ungetc
|
|
identify_identifier
|
|
else
|
|
ungetc
|
|
Token("$")
|
|
end
|
|
end
|
|
|
|
IDENT_RE = if defined? Encoding then
|
|
eval '/[\w\u{0080}-\u{FFFFF}]/u' # 1.8 can't parse \u{}
|
|
else
|
|
/[\w\x80-\xFF]/
|
|
end
|
|
|
|
def identify_identifier
|
|
token = ""
|
|
if peek(0) =~ /[$@]/
|
|
token.concat(c = getc)
|
|
if c == "@" and peek(0) == "@"
|
|
token.concat getc
|
|
end
|
|
end
|
|
|
|
while (ch = getc) =~ IDENT_RE do
|
|
print " :#{ch}: " if RDoc::RubyLex.debug?
|
|
token.concat ch
|
|
end
|
|
|
|
ungetc
|
|
|
|
if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
|
|
token.concat getc
|
|
end
|
|
|
|
# almost fix token
|
|
|
|
case token
|
|
when /^\$/
|
|
return Token(TkGVAR, token)
|
|
when /^\@\@/
|
|
@lex_state = :EXPR_END
|
|
# p Token(TkCVAR, token)
|
|
return Token(TkCVAR, token)
|
|
when /^\@/
|
|
@lex_state = :EXPR_END
|
|
return Token(TkIVAR, token)
|
|
end
|
|
|
|
if @lex_state != :EXPR_DOT
|
|
print token, "\n" if RDoc::RubyLex.debug?
|
|
|
|
token_c, *trans = TkReading2Token[token]
|
|
if token_c
|
|
# reserved word?
|
|
|
|
if (@lex_state != :EXPR_BEG &&
|
|
@lex_state != :EXPR_FNAME &&
|
|
trans[1])
|
|
# modifiers
|
|
token_c = TkSymbol2Token[trans[1]]
|
|
@lex_state = trans[0]
|
|
else
|
|
if @lex_state != :EXPR_FNAME
|
|
if ENINDENT_CLAUSE.include?(token)
|
|
valid = peek(0) != ':'
|
|
|
|
# check for ``class = val'' etc.
|
|
case token
|
|
when "class"
|
|
valid = false unless peek_match?(/^\s*(<<|\w|::)/)
|
|
when "def"
|
|
valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
|
|
when "do"
|
|
valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
|
|
when *ENINDENT_CLAUSE
|
|
valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
|
|
else
|
|
# no nothing
|
|
end if valid
|
|
|
|
if valid
|
|
if token == "do"
|
|
if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
|
|
@indent += 1
|
|
@indent_stack.push token_c
|
|
end
|
|
else
|
|
@indent += 1
|
|
@indent_stack.push token_c
|
|
end
|
|
else
|
|
token_c = TkIDENTIFIER
|
|
end
|
|
|
|
elsif DEINDENT_CLAUSE.include?(token)
|
|
@indent -= 1
|
|
@indent_stack.pop
|
|
end
|
|
@lex_state = trans[0]
|
|
else
|
|
@lex_state = :EXPR_END
|
|
end
|
|
end
|
|
return Token(token_c, token)
|
|
end
|
|
end
|
|
|
|
if @lex_state == :EXPR_FNAME
|
|
@lex_state = :EXPR_END
|
|
if peek(0) == '='
|
|
token.concat getc
|
|
end
|
|
elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
|
|
@lex_state == :EXPR_ARG
|
|
@lex_state = :EXPR_ARG
|
|
else
|
|
@lex_state = :EXPR_END
|
|
end
|
|
|
|
if token[0, 1] =~ /[A-Z]/
|
|
return Token(TkCONSTANT, token)
|
|
elsif token[token.size - 1, 1] =~ /[!?]/
|
|
return Token(TkFID, token)
|
|
else
|
|
return Token(TkIDENTIFIER, token)
|
|
end
|
|
end
|
|
|
|
def identify_here_document
|
|
ch = getc
|
|
# if lt = PERCENT_LTYPE[ch]
|
|
if ch == "-"
|
|
ch = getc
|
|
indent = true
|
|
end
|
|
if /['"`]/ =~ ch
|
|
user_quote = lt = ch
|
|
quoted = ""
|
|
while (c = getc) && c != lt
|
|
quoted.concat c
|
|
end
|
|
else
|
|
user_quote = nil
|
|
lt = '"'
|
|
quoted = ch.dup
|
|
while (c = getc) && c =~ /\w/
|
|
quoted.concat c
|
|
end
|
|
ungetc
|
|
end
|
|
|
|
ltback, @ltype = @ltype, lt
|
|
reserve = []
|
|
while ch = getc
|
|
reserve.push ch
|
|
if ch == "\\"
|
|
reserve.push ch = getc
|
|
elsif ch == "\n"
|
|
break
|
|
end
|
|
end
|
|
|
|
output_heredoc = reserve.join =~ /\A\r?\n\z/
|
|
|
|
if output_heredoc then
|
|
doc = '<<'
|
|
doc << '-' if indent
|
|
doc << "#{user_quote}#{quoted}#{user_quote}\n"
|
|
else
|
|
doc = '"'
|
|
end
|
|
|
|
@here_header = false
|
|
while l = gets
|
|
l = l.sub(/(:?\r)?\n\z/, "\n")
|
|
if (indent ? l.strip : l.chomp) == quoted
|
|
break
|
|
end
|
|
doc << l
|
|
end
|
|
|
|
if output_heredoc then
|
|
raise Error, "Missing terminating #{quoted} for string" unless l
|
|
|
|
doc << l.chomp
|
|
else
|
|
doc << '"'
|
|
end
|
|
|
|
@here_header = true
|
|
@here_readed.concat reserve
|
|
while ch = reserve.pop
|
|
ungetc ch
|
|
end
|
|
|
|
token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
|
|
@ltype = ltback
|
|
@lex_state = :EXPR_END
|
|
Token(token_class, doc)
|
|
end
|
|
|
|
def identify_quotation
|
|
type = ch = getc
|
|
if lt = PERCENT_LTYPE[type]
|
|
ch = getc
|
|
elsif type =~ /\W/
|
|
type = nil
|
|
lt = "\""
|
|
else
|
|
return Token(TkMOD, '%')
|
|
end
|
|
# if ch !~ /\W/
|
|
# ungetc
|
|
# next
|
|
# end
|
|
#@ltype = lt
|
|
@quoted = ch unless @quoted = PERCENT_PAREN[ch]
|
|
identify_string(lt, @quoted, type)
|
|
end
|
|
|
|
def identify_number(op = "")
|
|
@lex_state = :EXPR_END
|
|
|
|
num = op
|
|
|
|
if peek(0) == "0" && peek(1) !~ /[.eE]/
|
|
num << getc
|
|
|
|
case peek(0)
|
|
when /[xX]/
|
|
ch = getc
|
|
match = /[0-9a-fA-F_]/
|
|
when /[bB]/
|
|
ch = getc
|
|
match = /[01_]/
|
|
when /[oO]/
|
|
ch = getc
|
|
match = /[0-7_]/
|
|
when /[dD]/
|
|
ch = getc
|
|
match = /[0-9_]/
|
|
when /[0-7]/
|
|
match = /[0-7_]/
|
|
when /[89]/
|
|
raise Error, "Illegal octal digit"
|
|
else
|
|
return Token(TkINTEGER, num)
|
|
end
|
|
|
|
num << ch if ch
|
|
|
|
len0 = true
|
|
non_digit = false
|
|
while ch = getc
|
|
num << ch
|
|
if match =~ ch
|
|
if ch == "_"
|
|
if non_digit
|
|
raise Error, "trailing `#{ch}' in number"
|
|
else
|
|
non_digit = ch
|
|
end
|
|
else
|
|
non_digit = false
|
|
len0 = false
|
|
end
|
|
else
|
|
ungetc
|
|
num[-1, 1] = ''
|
|
if len0
|
|
raise Error, "numeric literal without digits"
|
|
end
|
|
if non_digit
|
|
raise Error, "trailing `#{non_digit}' in number"
|
|
end
|
|
break
|
|
end
|
|
end
|
|
return Token(TkINTEGER, num)
|
|
end
|
|
|
|
type = TkINTEGER
|
|
allow_point = true
|
|
allow_e = true
|
|
non_digit = false
|
|
while ch = getc
|
|
num << ch
|
|
case ch
|
|
when /[0-9]/
|
|
non_digit = false
|
|
when "_"
|
|
non_digit = ch
|
|
when allow_point && "."
|
|
if non_digit
|
|
raise Error, "trailing `#{non_digit}' in number"
|
|
end
|
|
type = TkFLOAT
|
|
if peek(0) !~ /[0-9]/
|
|
type = TkINTEGER
|
|
ungetc
|
|
num[-1, 1] = ''
|
|
break
|
|
end
|
|
allow_point = false
|
|
when allow_e && "e", allow_e && "E"
|
|
if non_digit
|
|
raise Error, "trailing `#{non_digit}' in number"
|
|
end
|
|
type = TkFLOAT
|
|
if peek(0) =~ /[+-]/
|
|
num << getc
|
|
end
|
|
allow_e = false
|
|
allow_point = false
|
|
non_digit = ch
|
|
else
|
|
if non_digit
|
|
raise Error, "trailing `#{non_digit}' in number"
|
|
end
|
|
ungetc
|
|
num[-1, 1] = ''
|
|
break
|
|
end
|
|
end
|
|
|
|
Token(type, num)
|
|
end
|
|
|
|
def identify_string(ltype, quoted = ltype, type = nil)
|
|
close = PERCENT_PAREN.values.include?(quoted)
|
|
@ltype = ltype
|
|
@quoted = quoted
|
|
|
|
str = if ltype == quoted and %w[" ' /].include? ltype then
|
|
ltype.dup
|
|
elsif RUBY_VERSION > '1.9' then
|
|
"%#{type or PERCENT_LTYPE.key ltype}#{PERCENT_PAREN_REV[quoted]||quoted}"
|
|
else
|
|
"%#{type or PERCENT_LTYPE.index ltype}#{PERCENT_PAREN_REV[quoted]||quoted}"
|
|
end
|
|
|
|
subtype = nil
|
|
begin
|
|
nest = 0
|
|
|
|
while ch = getc
|
|
str << ch
|
|
|
|
if @quoted == ch and nest <= 0
|
|
break
|
|
elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
|
|
ch = getc
|
|
subtype = true
|
|
if ch == "{" then
|
|
str << ch << skip_inner_expression
|
|
next
|
|
else
|
|
ungetc
|
|
end
|
|
elsif ch == '\\'
|
|
if %w[' /].include? @ltype then
|
|
case ch = getc
|
|
when "\\", "\n", "'"
|
|
when @ltype
|
|
str << ch
|
|
else
|
|
ungetc
|
|
end
|
|
else
|
|
str << read_escape
|
|
end
|
|
end
|
|
|
|
if close then
|
|
if PERCENT_PAREN[ch] == @quoted
|
|
nest += 1
|
|
elsif ch == @quoted
|
|
nest -= 1
|
|
end
|
|
end
|
|
end
|
|
|
|
if @ltype == "/"
|
|
while peek(0) =~ /i|m|x|o|e|s|u|n/
|
|
str << getc
|
|
end
|
|
end
|
|
|
|
if subtype
|
|
Token(DLtype2Token[ltype], str)
|
|
else
|
|
Token(Ltype2Token[ltype], str)
|
|
end
|
|
ensure
|
|
@ltype = nil
|
|
@quoted = nil
|
|
@lex_state = :EXPR_END
|
|
end
|
|
end
|
|
|
|
def skip_inner_expression
|
|
res = ""
|
|
nest = 0
|
|
while ch = getc
|
|
res << ch
|
|
if ch == '}'
|
|
break if nest.zero?
|
|
nest -= 1
|
|
elsif ch == '{'
|
|
nest += 1
|
|
end
|
|
end
|
|
res
|
|
end
|
|
|
|
def identify_comment
|
|
@ltype = "#"
|
|
|
|
comment = '#'
|
|
|
|
while ch = getc
|
|
# if ch == "\\" #"
|
|
# read_escape
|
|
# end
|
|
if ch == "\n"
|
|
@ltype = nil
|
|
ungetc
|
|
break
|
|
end
|
|
|
|
comment << ch
|
|
end
|
|
|
|
return Token(TkCOMMENT, comment)
|
|
end
|
|
|
|
def read_escape
|
|
escape = ''
|
|
ch = getc
|
|
|
|
case ch
|
|
when "\n", "\r", "\f"
|
|
escape << ch
|
|
when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
|
|
escape << ch
|
|
when /[0-7]/
|
|
ungetc ch
|
|
3.times do
|
|
ch = getc
|
|
case ch
|
|
when /[0-7]/
|
|
escape << ch
|
|
when nil
|
|
break
|
|
else
|
|
ungetc
|
|
break
|
|
end
|
|
end
|
|
|
|
when "x"
|
|
escape << ch
|
|
|
|
2.times do
|
|
ch = getc
|
|
case ch
|
|
when /[0-9a-fA-F]/
|
|
escape << ch
|
|
when nil
|
|
break
|
|
else
|
|
ungetc
|
|
break
|
|
end
|
|
end
|
|
|
|
when "M"
|
|
escape << ch
|
|
|
|
ch = getc
|
|
if ch != '-'
|
|
ungetc
|
|
else
|
|
escape << ch
|
|
|
|
ch = getc
|
|
if ch == "\\" #"
|
|
ungetc
|
|
escape << read_escape
|
|
else
|
|
escape << ch
|
|
end
|
|
end
|
|
|
|
when "C", "c" #, "^"
|
|
escape << ch
|
|
|
|
if ch == "C"
|
|
ch = getc
|
|
|
|
if ch == "-"
|
|
escape << ch
|
|
ch = getc
|
|
escape << ch
|
|
|
|
escape << read_escape if ch == "\\"
|
|
else
|
|
ungetc
|
|
end
|
|
elsif (ch = getc) == "\\" #"
|
|
escape << ch << read_escape
|
|
end
|
|
else
|
|
escape << ch
|
|
|
|
# other characters
|
|
end
|
|
|
|
escape
|
|
end
|
|
|
|
# :startdoc:
|
|
|
|
end
|
|
|
|
#RDoc::RubyLex.debug_level = 1
|
|
|