460 lines
11 KiB
Ruby
460 lines
11 KiB
Ruby
#--
|
|
# irb/ruby-token.rb - ruby tokens
|
|
# $Release Version: 0.9.5$
|
|
# $Revision: 11708 $
|
|
# $Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
|
|
# by Keiju ISHITSUKA(keiju@ruby-lang.org)
|
|
#++
|
|
# Definitions of all tokens involved in the lexical analysis.
|
|
#
|
|
# This class is not documented because it is so deep in the internals.
|
|
|
|
module RDoc::RubyToken
|
|
# :stopdoc:
|
|
|
|
EXPR_BEG = :EXPR_BEG
|
|
EXPR_MID = :EXPR_MID
|
|
EXPR_END = :EXPR_END
|
|
EXPR_ARG = :EXPR_ARG
|
|
EXPR_FNAME = :EXPR_FNAME
|
|
EXPR_DOT = :EXPR_DOT
|
|
EXPR_CLASS = :EXPR_CLASS
|
|
|
|
# for ruby 1.4X
|
|
if !defined?(Symbol)
|
|
Symbol = Integer
|
|
end
|
|
|
|
def set_token_position(line, char)
|
|
@prev_line_no = line
|
|
@prev_char_no = char
|
|
end
|
|
|
|
class Token
|
|
def initialize(seek, line_no, char_no, text = nil)
|
|
@seek = seek
|
|
@line_no = line_no
|
|
@char_no = char_no
|
|
@text = text
|
|
end
|
|
|
|
attr :seek
|
|
attr :line_no
|
|
attr :char_no
|
|
|
|
attr_accessor :text
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.text == @text
|
|
end
|
|
|
|
##
|
|
# Because we're used in contexts that expect to return a token, we set the
|
|
# text string and then return ourselves
|
|
|
|
def set_text(text)
|
|
@text = text
|
|
self
|
|
end
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text]
|
|
end
|
|
|
|
end
|
|
|
|
class TkNode < Token
|
|
def initialize(seek, line_no, char_no, node = nil)
|
|
super seek, line_no, char_no
|
|
@node = node
|
|
end
|
|
|
|
attr :node
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.node == @node
|
|
end
|
|
|
|
def set_text text
|
|
@node = text
|
|
self
|
|
end
|
|
|
|
alias text node
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node]
|
|
end
|
|
|
|
end
|
|
|
|
class TkId < Token
|
|
def initialize(seek, line_no, char_no, name)
|
|
super(seek, line_no, char_no)
|
|
@name = name
|
|
end
|
|
attr :name
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.name == @name
|
|
end
|
|
|
|
def set_text text
|
|
@name = text
|
|
self
|
|
end
|
|
|
|
alias text name
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
|
|
end
|
|
|
|
end
|
|
|
|
class TkKW < TkId
|
|
end
|
|
|
|
class TkVal < Token
|
|
def initialize(seek, line_no, char_no, value = nil)
|
|
super(seek, line_no, char_no)
|
|
@value = value
|
|
end
|
|
attr_accessor :value
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.value == @value
|
|
end
|
|
|
|
def set_text text
|
|
@value = text
|
|
self
|
|
end
|
|
|
|
alias text value
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %s, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value]
|
|
end
|
|
|
|
end
|
|
|
|
class TkOp < Token
|
|
def initialize(seek, line_no, char_no, name = nil)
|
|
super seek, line_no, char_no
|
|
@name = name
|
|
end
|
|
|
|
attr_accessor :name
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.name == @name
|
|
end
|
|
|
|
def set_text text
|
|
@name = text
|
|
self
|
|
end
|
|
|
|
alias text name
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
|
|
end
|
|
|
|
end
|
|
|
|
class TkOPASGN < TkOp
|
|
def initialize(seek, line_no, char_no, op)
|
|
super(seek, line_no, char_no)
|
|
op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
|
|
@op = op
|
|
@text = nil
|
|
end
|
|
|
|
attr :op
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.op == @op
|
|
end
|
|
|
|
def text
|
|
@text ||= "#{TkToken2Reading[op]}="
|
|
end
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op]
|
|
end
|
|
|
|
end
|
|
|
|
class TkUnknownChar < Token
|
|
def initialize(seek, line_no, char_no, name)
|
|
super(seek, line_no, char_no)
|
|
@name = name
|
|
end
|
|
attr :name
|
|
|
|
def ==(other)
|
|
self.class == other.class and
|
|
other.line_no == @line_no and
|
|
other.char_no == @char_no and
|
|
other.name == @name
|
|
end
|
|
|
|
def set_text text
|
|
@name = text
|
|
self
|
|
end
|
|
|
|
alias text name
|
|
|
|
def inspect # :nodoc:
|
|
klass = self.class.name.split('::').last
|
|
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
|
|
end
|
|
|
|
end
|
|
|
|
class TkError < Token
|
|
end
|
|
|
|
def Token(token, value = nil)
|
|
value ||= TkToken2Reading[token]
|
|
|
|
case token
|
|
when String
|
|
if (tk = TkReading2Token[token]).nil?
|
|
IRB.fail TkReading2TokenNoKey, token
|
|
end
|
|
|
|
tk = Token(tk[0], value)
|
|
|
|
if tk.kind_of?(TkOp) then
|
|
tk.name = token
|
|
end
|
|
when Symbol
|
|
if (tk = TkSymbol2Token[token]).nil?
|
|
IRB.fail TkSymbol2TokenNoKey, token
|
|
end
|
|
|
|
tk = Token(tk[0], value)
|
|
else
|
|
if token.instance_method(:initialize).arity == 3 then
|
|
tk = token.new(@prev_seek, @prev_line_no, @prev_char_no)
|
|
tk.set_text value
|
|
else
|
|
tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
|
|
end
|
|
end
|
|
|
|
tk
|
|
end
|
|
|
|
TokenDefinitions = [
|
|
[:TkCLASS, TkKW, "class", EXPR_CLASS],
|
|
[:TkMODULE, TkKW, "module", EXPR_BEG],
|
|
[:TkDEF, TkKW, "def", EXPR_FNAME],
|
|
[:TkUNDEF, TkKW, "undef", EXPR_FNAME],
|
|
[:TkBEGIN, TkKW, "begin", EXPR_BEG],
|
|
[:TkRESCUE, TkKW, "rescue", EXPR_MID],
|
|
[:TkENSURE, TkKW, "ensure", EXPR_BEG],
|
|
[:TkEND, TkKW, "end", EXPR_END],
|
|
[:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD],
|
|
[:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD],
|
|
[:TkTHEN, TkKW, "then", EXPR_BEG],
|
|
[:TkELSIF, TkKW, "elsif", EXPR_BEG],
|
|
[:TkELSE, TkKW, "else", EXPR_BEG],
|
|
[:TkCASE, TkKW, "case", EXPR_BEG],
|
|
[:TkWHEN, TkKW, "when", EXPR_BEG],
|
|
[:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD],
|
|
[:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD],
|
|
[:TkFOR, TkKW, "for", EXPR_BEG],
|
|
[:TkBREAK, TkKW, "break", EXPR_MID],
|
|
[:TkNEXT, TkKW, "next", EXPR_END],
|
|
[:TkREDO, TkKW, "redo", EXPR_END],
|
|
[:TkRETRY, TkKW, "retry", EXPR_END],
|
|
[:TkIN, TkKW, "in", EXPR_BEG],
|
|
[:TkDO, TkKW, "do", EXPR_BEG],
|
|
[:TkRETURN, TkKW, "return", EXPR_MID],
|
|
[:TkYIELD, TkKW, "yield", EXPR_END],
|
|
[:TkSUPER, TkKW, "super", EXPR_END],
|
|
[:TkSELF, TkKW, "self", EXPR_END],
|
|
[:TkNIL, TkKW, "nil", EXPR_END],
|
|
[:TkTRUE, TkKW, "true", EXPR_END],
|
|
[:TkFALSE, TkKW, "false", EXPR_END],
|
|
[:TkAND, TkKW, "and", EXPR_BEG],
|
|
[:TkOR, TkKW, "or", EXPR_BEG],
|
|
[:TkNOT, TkKW, "not", EXPR_BEG],
|
|
[:TkIF_MOD, TkKW],
|
|
[:TkUNLESS_MOD, TkKW],
|
|
[:TkWHILE_MOD, TkKW],
|
|
[:TkUNTIL_MOD, TkKW],
|
|
[:TkALIAS, TkKW, "alias", EXPR_FNAME],
|
|
[:TkDEFINED, TkKW, "defined?", EXPR_END],
|
|
[:TklBEGIN, TkKW, "BEGIN", EXPR_END],
|
|
[:TklEND, TkKW, "END", EXPR_END],
|
|
[:Tk__LINE__, TkKW, "__LINE__", EXPR_END],
|
|
[:Tk__FILE__, TkKW, "__FILE__", EXPR_END],
|
|
|
|
[:TkIDENTIFIER, TkId],
|
|
[:TkFID, TkId],
|
|
[:TkGVAR, TkId],
|
|
[:TkCVAR, TkId],
|
|
[:TkIVAR, TkId],
|
|
[:TkCONSTANT, TkId],
|
|
|
|
[:TkINTEGER, TkVal],
|
|
[:TkFLOAT, TkVal],
|
|
[:TkSTRING, TkVal],
|
|
[:TkHEREDOC, TkVal],
|
|
[:TkXSTRING, TkVal],
|
|
[:TkREGEXP, TkVal],
|
|
[:TkSYMBOL, TkVal],
|
|
|
|
[:TkDSTRING, TkNode],
|
|
[:TkDXSTRING, TkNode],
|
|
[:TkDREGEXP, TkNode],
|
|
[:TkNTH_REF, TkNode],
|
|
[:TkBACK_REF, TkNode],
|
|
|
|
[:TkUPLUS, TkOp, "+@"],
|
|
[:TkUMINUS, TkOp, "-@"],
|
|
[:TkPOW, TkOp, "**"],
|
|
[:TkCMP, TkOp, "<=>"],
|
|
[:TkEQ, TkOp, "=="],
|
|
[:TkEQQ, TkOp, "==="],
|
|
[:TkNEQ, TkOp, "!="],
|
|
[:TkGEQ, TkOp, ">="],
|
|
[:TkLEQ, TkOp, "<="],
|
|
[:TkANDOP, TkOp, "&&"],
|
|
[:TkOROP, TkOp, "||"],
|
|
[:TkMATCH, TkOp, "=~"],
|
|
[:TkNMATCH, TkOp, "!~"],
|
|
[:TkDOT2, TkOp, ".."],
|
|
[:TkDOT3, TkOp, "..."],
|
|
[:TkAREF, TkOp, "[]"],
|
|
[:TkASET, TkOp, "[]="],
|
|
[:TkLSHFT, TkOp, "<<"],
|
|
[:TkRSHFT, TkOp, ">>"],
|
|
[:TkCOLON2, TkOp, '::'],
|
|
[:TkCOLON3, TkOp, '::'],
|
|
#[:OPASGN, TkOp], # +=, -= etc. #
|
|
[:TkASSOC, TkOp, "=>"],
|
|
[:TkQUESTION, TkOp, "?"], #?
|
|
[:TkCOLON, TkOp, ":"], #:
|
|
|
|
[:TkfLPAREN, Token, "("], # func( #
|
|
[:TkfLBRACK, Token, "["], # func[ #
|
|
[:TkfLBRACE, Token, "{"], # func{ #
|
|
[:TkSYMBEG, Token, ":"], # :SYMBOL
|
|
|
|
[:TkAMPER, TkOp, "&"],
|
|
[:TkGT, TkOp, ">"],
|
|
[:TkLT, TkOp, "<"],
|
|
[:TkPLUS, TkOp, "+"],
|
|
[:TkSTAR, TkOp, "*"],
|
|
[:TkMINUS, TkOp, "-"],
|
|
[:TkMULT, TkOp, "*"],
|
|
[:TkDIV, TkOp, "/"],
|
|
[:TkMOD, TkOp, "%"],
|
|
[:TkBITOR, TkOp, "|"],
|
|
[:TkBITXOR, TkOp, "^"],
|
|
[:TkBITAND, TkOp, "&"],
|
|
[:TkBITNOT, TkOp, "~"],
|
|
[:TkNOTOP, TkOp, "!"],
|
|
|
|
[:TkBACKQUOTE, TkOp, "`"],
|
|
|
|
[:TkASSIGN, Token, "="],
|
|
[:TkDOT, Token, "."],
|
|
[:TkLPAREN, Token, "("], #(exp)
|
|
[:TkLBRACK, Token, "["], #[arry]
|
|
[:TkLBRACE, Token, "{"], #{hash}
|
|
[:TkRPAREN, Token, ")"],
|
|
[:TkRBRACK, Token, "]"],
|
|
[:TkRBRACE, Token, "}"],
|
|
[:TkCOMMA, Token, ","],
|
|
[:TkSEMICOLON, Token, ";"],
|
|
|
|
[:TkCOMMENT, TkVal],
|
|
[:TkSPACE, Token, " "],
|
|
[:TkNL, Token, "\n"],
|
|
[:TkEND_OF_SCRIPT],
|
|
|
|
[:TkBACKSLASH, TkUnknownChar, "\\"],
|
|
[:TkAT, TkUnknownChar, "@"],
|
|
[:TkDOLLAR, TkUnknownChar, "$"],
|
|
]
|
|
|
|
# {reading => token_class}
|
|
# {reading => [token_class, *opt]}
|
|
TkReading2Token = {}
|
|
TkToken2Reading = {}
|
|
TkSymbol2Token = {}
|
|
|
|
def self.def_token(token_n, super_token = Token, reading = nil, *opts)
|
|
token_n = token_n.id2name if token_n.kind_of?(Symbol)
|
|
if const_defined?(token_n)
|
|
IRB.fail AlreadyDefinedToken, token_n
|
|
end
|
|
token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
|
|
|
|
if reading
|
|
TkToken2Reading[token_c] = reading
|
|
|
|
return if TkReading2Token[reading]
|
|
|
|
if opts.empty?
|
|
TkReading2Token[reading] = [token_c]
|
|
else
|
|
TkReading2Token[reading] = [token_c].concat(opts)
|
|
end
|
|
end
|
|
TkSymbol2Token[token_n.intern] = token_c
|
|
end
|
|
|
|
for defs in TokenDefinitions
|
|
def_token(*defs)
|
|
end
|
|
|
|
def_token :TkRD_COMMENT, TkCOMMENT
|
|
|
|
NEWLINE_TOKEN = TkNL.new 0, 0, 0, "\n"
|
|
|
|
class TkSYMBOL
|
|
|
|
def to_sym
|
|
@sym ||= text[1..-1].intern
|
|
end
|
|
|
|
end
|
|
|
|
# :startdoc:
|
|
end
|
|
|