Module: RDoc::RubyToken

Included in:
Parser::Ruby, Parser::RubyTools, RubyLex
Defined in:
lib/rdoc/ruby_token.rb

Overview

–

irb/ruby-token.rb - ruby tokens
	$Release Version: 0.9.5$
	$Revision: 11708 $
	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
	by Keiju ISHITSUKA([email protected])

++ Definitions of all tokens involved in the lexical analysis.

This class is not documented because it is so deep in the internals.

Defined Under Namespace

Classes: TkError, TkId, TkKW, TkNode, TkOPASGN, TkOp, TkSYMBOL, TkUnknownChar, TkVal, Token

Constant Summary collapse

EXPR_BEG =

:stopdoc:

:EXPR_BEG
EXPR_MID = 
:EXPR_MID
EXPR_END = 
:EXPR_END
EXPR_ARG = 
:EXPR_ARG
EXPR_FNAME = 
:EXPR_FNAME
EXPR_DOT = 
:EXPR_DOT
EXPR_CLASS = 
:EXPR_CLASS
Symbol = 
Integer
TokenDefinitions = 
[
  [:TkCLASS,      TkKW,  "class",  :EXPR_CLASS],
  [:TkMODULE,     TkKW,  "module", :EXPR_BEG],
  [:TkDEF,        TkKW,  "def",    :EXPR_FNAME],
  [:TkUNDEF,      TkKW,  "undef",  :EXPR_FNAME],
  [:TkBEGIN,      TkKW,  "begin",  :EXPR_BEG],
  [:TkRESCUE,     TkKW,  "rescue", :EXPR_MID],
  [:TkENSURE,     TkKW,  "ensure", :EXPR_BEG],
  [:TkEND,        TkKW,  "end",    :EXPR_END],
  [:TkIF,         TkKW,  "if",     :EXPR_BEG, :TkIF_MOD],
  [:TkUNLESS,     TkKW,  "unless", :EXPR_BEG, :TkUNLESS_MOD],
  [:TkTHEN,       TkKW,  "then",   :EXPR_BEG],
  [:TkELSIF,      TkKW,  "elsif",  :EXPR_BEG],
  [:TkELSE,       TkKW,  "else",   :EXPR_BEG],
  [:TkCASE,       TkKW,  "case",   :EXPR_BEG],
  [:TkWHEN,       TkKW,  "when",   :EXPR_BEG],
  [:TkWHILE,      TkKW,  "while",  :EXPR_BEG, :TkWHILE_MOD],
  [:TkUNTIL,      TkKW,  "until",  :EXPR_BEG, :TkUNTIL_MOD],
  [:TkFOR,        TkKW,  "for",    :EXPR_BEG],
  [:TkBREAK,      TkKW,  "break",  :EXPR_MID],
  [:TkNEXT,       TkKW,  "next",   :EXPR_END],
  [:TkREDO,       TkKW,  "redo",   :EXPR_END],
  [:TkRETRY,      TkKW,  "retry",  :EXPR_END],
  [:TkIN,         TkKW,  "in",     :EXPR_BEG],
  [:TkDO,         TkKW,  "do",     :EXPR_BEG],
  [:TkRETURN,     TkKW,  "return", :EXPR_MID],
  [:TkYIELD,      TkKW,  "yield",  :EXPR_END],
  [:TkSUPER,      TkKW,  "super",  :EXPR_END],
  [:TkSELF,       TkKW,  "self",   :EXPR_END],
  [:TkNIL,        TkKW,  "nil",    :EXPR_END],
  [:TkTRUE,       TkKW,  "true",   :EXPR_END],
  [:TkFALSE,      TkKW,  "false",  :EXPR_END],
  [:TkAND,        TkKW,  "and",    :EXPR_BEG],
  [:TkOR,         TkKW,  "or",     :EXPR_BEG],
  [:TkNOT,        TkKW,  "not",    :EXPR_BEG],
  [:TkIF_MOD,     TkKW],
  [:TkUNLESS_MOD, TkKW],
  [:TkWHILE_MOD,  TkKW],
  [:TkUNTIL_MOD,  TkKW],
  [:TkALIAS,      TkKW,  "alias",    :EXPR_FNAME],
  [:TkDEFINED,    TkKW,  "defined?", :EXPR_END],
  [:TklBEGIN,     TkKW,  "BEGIN",    :EXPR_END],
  [:TklEND,       TkKW,  "END",      :EXPR_END],
  [:Tk__LINE__,   TkKW,  "__LINE__", :EXPR_END],
  [:Tk__FILE__,   TkKW,  "__FILE__", :EXPR_END],

  [:TkIDENTIFIER, TkId],
  [:TkFID,        TkId],
  [:TkGVAR,       TkId],
  [:TkCVAR,       TkId],
  [:TkIVAR,       TkId],
  [:TkCONSTANT,   TkId],

  [:TkINTEGER,    TkVal],
  [:TkFLOAT,      TkVal],
  [:TkSTRING,     TkVal],
  [:TkHEREDOC,    TkVal],
  [:TkXSTRING,    TkVal],
  [:TkREGEXP,     TkVal],
  [:TkSYMBOL,     TkVal],
  [:TkCHAR,       TkVal],

  [:TkDSTRING,    TkNode],
  [:TkDXSTRING,   TkNode],
  [:TkDREGEXP,    TkNode],
  [:TkNTH_REF,    TkNode],
  [:TkBACK_REF,   TkNode],

  [:TkUPLUS,      TkOp,   "+@"],
  [:TkUMINUS,     TkOp,   "-@"],
  [:TkPOW,        TkOp,   "**"],
  [:TkCMP,        TkOp,   "<=>"],
  [:TkEQ,         TkOp,   "=="],
  [:TkEQQ,        TkOp,   "==="],
  [:TkNEQ,        TkOp,   "!="],
  [:TkGEQ,        TkOp,   ">="],
  [:TkLEQ,        TkOp,   "<="],
  [:TkANDOP,      TkOp,   "&&"],
  [:TkOROP,       TkOp,   "||"],
  [:TkMATCH,      TkOp,   "=~"],
  [:TkNMATCH,     TkOp,   "!~"],
  [:TkDOT2,       TkOp,   ".."],
  [:TkDOT3,       TkOp,   "..."],
  [:TkAREF,       TkOp,   "[]"],
  [:TkASET,       TkOp,   "[]="],
  [:TkLSHFT,      TkOp,   "<<"],
  [:TkRSHFT,      TkOp,   ">>"],
  [:TkCOLON2,     TkOp,   '::'],
  [:TkCOLON3,     TkOp,   '::'],
  #[:OPASGN,       TkOp],               # +=, -=  etc. #
  [:TkASSOC,      TkOp,   "=>"],
  [:TkQUESTION,   TkOp,   "?"], #?
  [:TkCOLON,      TkOp,   ":"],        #:

  [:TkfLPAREN,    Token,  "("], # func( #
  [:TkfLBRACK,    Token,  "["], # func[ #
  [:TkfLBRACE,    Token,  "{"], # func{ #
  [:TkSYMBEG,     Token,  ":"], # :SYMBOL

  [:TkAMPER,      TkOp,   "&"],
  [:TkGT,         TkOp,   ">"],
  [:TkLT,         TkOp,   "<"],
  [:TkPLUS,       TkOp,   "+"],
  [:TkSTAR,       TkOp,   "*"],
  [:TkMINUS,      TkOp,   "-"],
  [:TkMULT,       TkOp,   "*"],
  [:TkDIV,        TkOp,   "/"],
  [:TkMOD,        TkOp,   "%"],
  [:TkBITOR,      TkOp,   "|"],
  [:TkBITXOR,     TkOp,   "^"],
  [:TkBITAND,     TkOp,   "&"],
  [:TkBITNOT,     TkOp,   "~"],
  [:TkNOTOP,      TkOp,   "!"],

  [:TkBACKQUOTE,  TkOp,   "`"],

  [:TkASSIGN,     Token,  "="],
  [:TkDOT,        Token,  "."],
  [:TkLPAREN,     Token,  "("],  #(exp)
  [:TkLBRACK,     Token,  "["],  #[arry]
  [:TkLBRACE,     Token,  "{"],  #{hash}
  [:TkRPAREN,     Token,  ")"],
  [:TkRBRACK,     Token,  "]"],
  [:TkRBRACE,     Token,  "}"],
  [:TkCOMMA,      Token,  ","],
  [:TkSEMICOLON,  Token,  ";"],

  [:TkCOMMENT,    TkVal],
  [:TkSPACE,      Token,  " "],
  [:TkNL,         Token,  "\n"],
  [:TkEND_OF_SCRIPT],

  [:TkBACKSLASH,  TkUnknownChar,  "\\"],
  [:TkAT,         TkUnknownChar,  "@"],
  [:TkDOLLAR,     TkUnknownChar,  "$"],
]
TkReading2Token =

=> token_class => [token_class, *opt]

{}
TkToken2Reading = 
{}
TkSymbol2Token = 
{}
NEWLINE_TOKEN = 
TkNL.new 0, 0, 0, "\n"

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.def_token(token_n, super_token = Token, reading = nil, *opts) ⇒ Object 



421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
 
# File 'lib/rdoc/ruby_token.rb', line 421

def self.def_token(token_n, super_token = Token, reading = nil, *opts)
  token_n = token_n.id2name if token_n.kind_of?(Symbol)
  if const_defined?(token_n)
    IRB.fail AlreadyDefinedToken, token_n
  end
  token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")

  if reading
    TkToken2Reading[token_c] = reading

    return if TkReading2Token[reading]

    if opts.empty?
      TkReading2Token[reading] = [token_c]
    else
      TkReading2Token[reading] = [token_c].concat(opts)
    end
  end
  TkSymbol2Token[token_n.intern] = token_c
end

Instance Method Details

#set_token_position(line, char) ⇒ Object 



28
29
30
31
 
# File 'lib/rdoc/ruby_token.rb', line 28

def set_token_position(line, char)
  @prev_line_no = line
  @prev_char_no = char
end

#Token(token, value = nil) ⇒ Object 



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/rdoc/ruby_token.rb', line 246

def Token(token, value = nil)
  value ||= TkToken2Reading[token]

  case token
  when String
    if (tk = TkReading2Token[token]).nil?
      IRB.fail TkReading2TokenNoKey, token
    end

    tk = Token(tk[0], value)

    if tk.kind_of?(TkOp) then
      tk.name = token
    end
  when Symbol
    if (tk = TkSymbol2Token[token]).nil?
      IRB.fail TkSymbol2TokenNoKey, token
    end

    tk = Token(tk[0], value)
  else
    if token.instance_method(:initialize).arity == 3 then
      tk = token.new(@prev_seek, @prev_line_no, @prev_char_no)
      tk.set_text value
    else
      tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
    end
  end

  tk
end