Class: Ionize::Php::Tokenizer

Inherits:
Dhaka::Tokenizer
  • Object
show all
Defined in:
lib/ionize/tokenizer.rb

Constant Summary collapse

Symbols = 
['[', ']', '(', ')', '{', '}', ':', ';', ',', '!', '-']
AllCharacters =

Why did I do it this way? Might be easier to read as just straight symbols

digits + Symbols + whitespace + star +
underscore + squotes + dquotes + letters + newline + dot + 
tab + equals + dollar + plus + minus + at +
pipe + greater + less_than + percent + ampersand +
["^", "?", '#', "\r", "/", "\\"]
Keywords = 
%w(as and break case class default do echo 
extends else exit for foreach function global 
if include include_once list new or print 
protected private require require_once return 
static switch var while)
Whitespace = 
[" ", "\t", "\r", "\n"]
Identifiers = 
letters + underscore + digits
NonIdentifiers = 
Whitespace + ['"', "'", ">", "<", "="] + Symbols + 
["^", "+", "-", "|", ".", "/", "&", "\n", "\\", "$"]

Instance Method Summary collapse

Instance Method Details

#create_cast(cast) ⇒ Object 



82
83
84
85
 
# File 'lib/ionize/tokenizer.rb', line 82

def create_cast cast
  create_token 'cast', cast
  cast.length.times { advance }    
end

#create_keyword(keyword) ⇒ Object 



77
78
79
80
 
# File 'lib/ionize/tokenizer.rb', line 77

def create_keyword keyword
  create_token keyword, keyword
  keyword.length.times { advance }    
end

#double_peek ⇒ Object 



65
66
67
 
# File 'lib/ionize/tokenizer.rb', line 65

def double_peek
  @input[@curr_char_index+2] and @input[@curr_char_index+2].chr 
end

#double_reverse_peek ⇒ Object 



73
74
75
 
# File 'lib/ionize/tokenizer.rb', line 73

def double_reverse_peek
  @input[@curr_char_index-2] and @input[@curr_char_index-2].chr 
end

#exactly_matches?(keyword) ⇒ Boolean

Returns:

  • (Boolean) 


87
88
89
 
# File 'lib/ionize/tokenizer.rb', line 87

def exactly_matches? keyword
  @input.slice(@curr_char_index, keyword.length) == keyword
end

#peek ⇒ Object

Just have to figure out Dhaka’s own regex engine



61
62
63
 
# File 'lib/ionize/tokenizer.rb', line 61

def peek
  @input[@curr_char_index+1] and @input[@curr_char_index+1].chr 
end

#reverse_peek ⇒ Object 



69
70
71
 
# File 'lib/ionize/tokenizer.rb', line 69

def reverse_peek
  @input[@curr_char_index-1] and @input[@curr_char_index-1].chr 
end