Class: Dhaka::LexerRun

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
lib/dhaka/lexer/lexer_run.rb

Overview

Represents a run of a lexer on a given input string.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(lexer, input) ⇒ LexerRun

Returns a new instance of LexerRun.



7
8
9
10
11
12
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 7

def initialize lexer, input
  @lexer, @input          = lexer, input
  @input_position         = 0
  @not_yet_accepted_chars = []
  @last_saved_checkpoints = {}
end

Instance Attribute Details

#current_lexeme ⇒ Object (readonly)

Returns the value of attribute current_lexeme.



6
7
8
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 6

def current_lexeme
  @current_lexeme
end

Instance Method Details

#accept(pattern) ⇒ Object

:nodoc:



41
42
43
44
45
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 41

def accept(pattern) #:nodoc:
  @current_lexeme.pattern = pattern
  @current_lexeme.concat @not_yet_accepted_chars
  @not_yet_accepted_chars = []
end

#accept_last_saved_checkpoint(pattern) ⇒ Object

:nodoc:



51
52
53
54
55
56
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 51

def accept_last_saved_checkpoint(pattern) #:nodoc:
  @current_lexeme.pattern = pattern
  @current_lexeme.concat @not_yet_accepted_chars
  @not_yet_accepted_chars = @current_lexeme.characters[(@last_saved_checkpoints[pattern].size)..-1]
  @current_lexeme.characters = @last_saved_checkpoints[pattern].dup
end

#create_token(symbol_name, value = current_lexeme.characters.join) ⇒ Object

Constructs a token of type symbol_name from the current_lexeme.



15
16
17
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 15

def create_token(symbol_name, value = current_lexeme.characters.join)
  Token.new(symbol_name, value, current_lexeme.input_position)
end

#each {|Token.new(END_SYMBOL_NAME, nil, nil)| ... } ⇒ Object

Yields each token as it is recognized. Returns a TokenizerErrorResult if an error occurs during tokenization.

Yields:



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 20

def each
  reset_and_rewind
  loop do
    c = curr_char
    break if (c == "\0" && @not_yet_accepted_chars.empty? && !@current_lexeme.accepted?)
    dest_state  = @curr_state.transitions[c]
    unless dest_state
      return TokenizerErrorResult.new(@input_position) unless @current_lexeme.accepted?
      token = get_token
      yield token if token
      reset_and_rewind
    else
      @curr_state = dest_state
      @not_yet_accepted_chars << c
      @curr_state.process(self) 
      advance
    end
  end
  yield Token.new(END_SYMBOL_NAME, nil, nil)
end

#save_checkpoint(pattern) ⇒ Object

:nodoc:



47
48
49
 
# File 'lib/dhaka/lexer/lexer_run.rb', line 47

def save_checkpoint(pattern) #:nodoc:
  @last_saved_checkpoints[pattern] = (@current_lexeme.characters + @not_yet_accepted_chars)
end