Class: Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ Tokenizer

Returns a new instance of Tokenizer.



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/tokenizer.rb', line 11

def initialize text
  @stream = text
  @tokens = []

  skip_whitespace
  while tokenizing?
    if operator?(current_char)
      if '.:'.include?(next_char)
        @tokens << current_char + next_char
        advance(2)
      else
        @tokens << current_char
        advance(1)
      end
    else
      if stream =~ /^_?\d+/
        match_data = /^_?\d+/.match(stream)
        @tokens << match_data[0]
        advance(match_data[0].length)
      else
        break
      end
    end
    skip_whitespace
  end
end

Instance Attribute Details

#stream ⇒ Object (readonly)

Returns the value of attribute stream.



5
6
7
 
# File 'lib/tokenizer.rb', line 5

def stream
  @stream
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



5
6
7
 
# File 'lib/tokenizer.rb', line 5

def tokens
  @tokens
end

Instance Method Details

#operator?(char) ⇒ Boolean

Returns:

  • (Boolean) 


7
8
9
 
# File 'lib/tokenizer.rb', line 7

def operator? char
  '~^-%*|<>\\{}+/#$'.include?(char)
end