Class: Dhaka::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tokenizer/tokenizer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ Tokenizer

:nodoc:



127
128
129
130
131
132
 
# File 'lib/tokenizer/tokenizer.rb', line 127

def initialize(input) #:nodoc:
  @input = input
  @current_state = self.class.states[TOKENIZER_IDLE_STATE]
  @curr_char_index = 0
  @tokens = []
end

Instance Attribute Details

#tokens ⇒ Object (readonly)

The tokens shifted so far.



125
126
127
 
# File 'lib/tokenizer/tokenizer.rb', line 125

def tokens
  @tokens
end

Class Method Details

.for_state(state_name, &blk) ⇒ Object

Define the action for the state named state_name.



115
116
117
 
# File 'lib/tokenizer/tokenizer.rb', line 115

def self.for_state(state_name, &blk)
  states[state_name].instance_eval(&blk)
end

.tokenize(input) ⇒ Object

Tokenizes a string input and returns a TokenizerErrorResult on failure or a TokenizerSuccessResult on sucess.



120
121
122
 
# File 'lib/tokenizer/tokenizer.rb', line 120

def self.tokenize(input)
  self.new(input).run
end

Instance Method Details

#advance ⇒ Object

Advance to the next character.



140
141
142
 
# File 'lib/tokenizer/tokenizer.rb', line 140

def advance
  @curr_char_index += 1
end

#create_token(symbol_name, value) ⇒ Object

Push a new token on to the stack with symbol corresponding to symbol_name and a value of value.



150
151
152
153
 
# File 'lib/tokenizer/tokenizer.rb', line 150

def create_token(symbol_name, value)
  new_token = Dhaka::Token.new(symbol_name, value, @curr_char_index)
  tokens << new_token
end

#curr_char ⇒ Object

The character currently being processed.



135
136
137
 
# File 'lib/tokenizer/tokenizer.rb', line 135

def curr_char
  @input[@curr_char_index] and @input[@curr_char_index].chr 
end

#curr_token ⇒ Object

The token currently on top of the stack.



145
146
147
 
# File 'lib/tokenizer/tokenizer.rb', line 145

def curr_token
  tokens[-1]
end

#run ⇒ Object

:nodoc:



160
161
162
163
164
165
166
167
168
 
# File 'lib/tokenizer/tokenizer.rb', line 160

def run #:nodoc:
  while curr_char
    blk = @current_state.actions[curr_char]
    return TokenizerErrorResult.new(@curr_char_index) unless blk
    instance_eval(&blk)
  end
  tokens << Dhaka::Token.new(Dhaka::END_SYMBOL_NAME, nil, nil)
  return TokenizerSuccessResult.new(tokens)
end

#switch_to(state_name) ⇒ Object

Change the active state of the tokenizer to the state identified by the symbol state_name.



156
157
158
 
# File 'lib/tokenizer/tokenizer.rb', line 156

def switch_to state_name
  @current_state = self.class.states[state_name]
end