Module: RubyToken

Included in:
RDocF95::RubyParser, RubyLex
Defined in:
lib/rdoc-f95/parsers/parse_rb.rb

Overview

Definitions of all tokens involved in the lexical analysis

Defined Under Namespace

Classes: TkError, TkId, TkKW, TkNode, TkOPASGN, TkOp, TkUnknownChar, TkVal, Token

Constant Summary collapse

EXPR_BEG = 
:EXPR_BEG
EXPR_MID = 
:EXPR_MID
EXPR_END = 
:EXPR_END
EXPR_ARG = 
:EXPR_ARG
EXPR_FNAME = 
:EXPR_FNAME
EXPR_DOT = 
:EXPR_DOT
EXPR_CLASS = 
:EXPR_CLASS
TokenDefinitions = 
[
  [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
  [:TkMODULE,     TkKW,  "module", EXPR_BEG],
  [:TkDEF,	    TkKW,  "def",    EXPR_FNAME],
  [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
  [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
  [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
  [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
  [:TkEND,	    TkKW,  "end",    EXPR_END],
  [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
  [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
  [:TkTHEN,	    TkKW,  "then",   EXPR_BEG],
  [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
  [:TkELSE,	    TkKW,  "else",   EXPR_BEG],
  [:TkCASE,	    TkKW,  "case",   EXPR_BEG],
  [:TkWHEN,	    TkKW,  "when",   EXPR_BEG],
  [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
  [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
  [:TkFOR,	    TkKW,  "for",    EXPR_BEG],
  [:TkBREAK,      TkKW,  "break",  EXPR_END],
  [:TkNEXT,	    TkKW,  "next",   EXPR_END],
  [:TkREDO,	    TkKW,  "redo",   EXPR_END],
  [:TkRETRY,      TkKW,  "retry",  EXPR_END],
  [:TkIN,	    TkKW,  "in",     EXPR_BEG],
  [:TkDO,	    TkKW,  "do",     EXPR_BEG],
  [:TkRETURN,     TkKW,  "return", EXPR_MID],
  [:TkYIELD,      TkKW,  "yield",  EXPR_END],
  [:TkSUPER,      TkKW,  "super",  EXPR_END],
  [:TkSELF,	    TkKW,  "self",   EXPR_END],
  [:TkNIL, 	    TkKW,  "nil",    EXPR_END],
  [:TkTRUE,	    TkKW,  "true",   EXPR_END],
  [:TkFALSE,      TkKW,  "false",  EXPR_END],
  [:TkAND,	    TkKW,  "and",    EXPR_BEG],
  [:TkOR, 	    TkKW,  "or",     EXPR_BEG],
  [:TkNOT,	    TkKW,  "not",    EXPR_BEG],
  [:TkIF_MOD,     TkKW],
  [:TkUNLESS_MOD, TkKW],
  [:TkWHILE_MOD,  TkKW],
  [:TkUNTIL_MOD,  TkKW],
  [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
  [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
  [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
  [:TklEND,	    TkKW,  "END",      EXPR_END],
  [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
  [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],

  [:TkIDENTIFIER, TkId],
  [:TkFID,	    TkId],
  [:TkGVAR,	    TkId],
  [:TkIVAR,	    TkId],
  [:TkCONSTANT,   TkId],

  [:TkINTEGER,    TkVal],
  [:TkFLOAT,      TkVal],
  [:TkSTRING,     TkVal],
  [:TkXSTRING,    TkVal],
  [:TkREGEXP,     TkVal],
  [:TkCOMMENT,    TkVal],

  [:TkDSTRING,    TkNode],
  [:TkDXSTRING,   TkNode],
  [:TkDREGEXP,    TkNode],
  [:TkNTH_REF,    TkId],
  [:TkBACK_REF,   TkId],

  [:TkUPLUS,      TkOp,   "+@"],
  [:TkUMINUS,     TkOp,   "-@"],
  [:TkPOW,	    TkOp,   "**"],
  [:TkCMP,	    TkOp,   "<=>"],
  [:TkEQ,	    TkOp,   "=="],
  [:TkEQQ,	    TkOp,   "==="],
  [:TkNEQ,	    TkOp,   "!="],
  [:TkGEQ,	    TkOp,   ">="],
  [:TkLEQ,	    TkOp,   "<="],
  [:TkANDOP,      TkOp,   "&&"],
  [:TkOROP,	    TkOp,   "||"],
  [:TkMATCH,      TkOp,   "=~"],
  [:TkNMATCH,     TkOp,   "!~"],
  [:TkDOT2,	    TkOp,   ".."],
  [:TkDOT3,	    TkOp,   "..."],
  [:TkAREF,	    TkOp,   "[]"],
  [:TkASET,	    TkOp,   "[]="],
  [:TkLSHFT,      TkOp,   "<<"],
  [:TkRSHFT,      TkOp,   ">>"],
  [:TkCOLON2,     TkOp],
  [:TkCOLON3,     TkOp],
#   [:OPASGN,	    TkOp],               # +=, -=  etc. #
  [:TkASSOC,      TkOp,   "=>"],
  [:TkQUESTION,   TkOp,   "?"],	 #?
  [:TkCOLON,      TkOp,   ":"],        #:

  [:TkfLPAREN],         # func( #
  [:TkfLBRACK],         # func[ #
  [:TkfLBRACE],         # func{ #
  [:TkSTAR],            # *arg
  [:TkAMPER],           # &arg #
  [:TkSYMBOL,     TkId],          # :SYMBOL
  [:TkSYMBEG,     TkId],
  [:TkGT,	    TkOp,   ">"],
  [:TkLT,	    TkOp,   "<"],
  [:TkPLUS,	    TkOp,   "+"],
  [:TkMINUS,      TkOp,   "-"],
  [:TkMULT,	    TkOp,   "*"],
  [:TkDIV,	    TkOp,   "/"],
  [:TkMOD,	    TkOp,   "%"],
  [:TkBITOR,      TkOp,   "|"],
  [:TkBITXOR,     TkOp,   "^"],
  [:TkBITAND,     TkOp,   "&"],
  [:TkBITNOT,     TkOp,   "~"],
  [:TkNOTOP,      TkOp,   "!"],

  [:TkBACKQUOTE,  TkOp,   "`"],

  [:TkASSIGN,     Token,  "="],
  [:TkDOT,	    Token,  "."],
  [:TkLPAREN,     Token,  "("],  #(exp)
  [:TkLBRACK,     Token,  "["],  #[arry]
  [:TkLBRACE,     Token,  "{"],  #{hash}
  [:TkRPAREN,     Token,  ")"],
  [:TkRBRACK,     Token,  "]"],
  [:TkRBRACE,     Token,  "}"],
  [:TkCOMMA,      Token,  ","],
  [:TkSEMICOLON,  Token,  ";"],

  [:TkRD_COMMENT],
  [:TkSPACE],
  [:TkNL],
  [:TkEND_OF_SCRIPT],

  [:TkBACKSLASH,  TkUnknownChar,  "\\"],
  [:TkAT,	    TkUnknownChar,  "@"],
  [:TkDOLLAR,     TkUnknownChar,  "\$"], #"
]
TkReading2Token =

=> token_class => [token_class, *opt]

{}
TkSymbol2Token = 
{}
NEWLINE_TOKEN = 
TkNL.new(0,0)

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.def_token(token_n, super_token = Token, reading = nil, *opts) ⇒ Object 



270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/rdoc-f95/parsers/parse_rb.rb', line 270

def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
  token_n = token_n.id2name unless token_n.kind_of?(String)
  if RubyToken.const_defined?(token_n)
    fail AlreadyDefinedToken, token_n
  end

  token_c =  Class.new super_token
  RubyToken.const_set token_n, token_c
#    token_c.inspect

  if reading
    if TkReading2Token[reading]
      fail TkReading2TokenDuplicateError, token_n, reading
    end
    if opts.empty?
	TkReading2Token[reading] = [token_c]
    else
	TkReading2Token[reading] = [token_c].concat(opts)
    end
  end
  TkSymbol2Token[token_n.intern] = token_c

  if token_c <= TkOp
    token_c.class_eval %{
      def self.op_name; "#{reading}"; end
    }
  end
end

Instance Method Details

#set_token_position(line, char) ⇒ Object 



107
108
109
110
 
# File 'lib/rdoc-f95/parsers/parse_rb.rb', line 107

def set_token_position(line, char)
  @prev_line_no = line
  @prev_char_no = char
end

#Token(token, value = nil) ⇒ Object 



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/rdoc-f95/parsers/parse_rb.rb', line 112

def Token(token, value = nil)
  tk = nil
  case token
  when String, Symbol
    source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
    if (tk = source[token]).nil?
      fail TkReading2TokenNoKey, token
    end
    tk = Token(tk[0], value)
  else
    tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
           token.new(@prev_line_no, @prev_char_no)
         else
           token.new(@prev_line_no, @prev_char_no, value)
         end
  end
  tk
end