Class: Rgo::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/rgo/tokenizer.rb

Defined Under Namespace

Modules: Expression

Instance Method Summary collapse

Constructor Details

#initialize(source) ⇒ Tokenizer

Returns a new instance of Tokenizer.



5
6
7
 
# File 'lib/rgo/tokenizer.rb', line 5

def initialize(source)
  @source = source
end

Instance Method Details

#tokens ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/rgo/tokenizer.rb', line 9

def tokens
  @q = []
  scanner = StringScanner.new(@source)

  until scanner.empty?
    case
      when m=scanner.scan(/\s+/)
        blank_lines = m.count("\n") - 1
        blank_lines.times do
          @q.push [:BLANK_LINE, ""]
        end
      when m = scanner.scan(Expression::INTEGER)
        @q.push [:INTEGER, m.to_i]
      when m = scanner.scan(Expression::KEYWORDS)
        @q.push [:"KEYWORD_#{m.upcase}", m]
      when m = scanner.scan(Expression::COMMENT)
        @q.push [:COMMENT, m[1..-1]]
      when m = scanner.scan(Expression::STRING)
        @q.push [:STRING, m[1..-2]]
      when m = scanner.scan(Expression::CONSTANT)
        @q.push [:CONSTANT, m]
      when m = scanner.scan(Expression::IDENTIFIER)
        @q.push [:IDENTIFIER, m]
      when m = scanner.scan(/==/)
        @q.push [:EQUAL, "=="]
      when m = scanner.scan(/=/)
        @q.push [:KEYWORD_ASSIGN, "="]
      when m = scanner.scan(/\+/)
        @q.push [:PLUS, "+"]
      when m = scanner.scan(/\-/)
        @q.push [:MINUS, "-"]
      when m = scanner.scan(/\*/)
        @q.push [:MULTIPLY, "*"]
      when m = scanner.scan(/\//)
        @q.push [:DIVIDE, "/"]
      when m = scanner.scan(/>/)
        @q.push [:GREATER, ">"]
      when m = scanner.scan(/</)
        @q.push [:LESS, "<"]
      when m = scanner.scan(/\%/)
        @q.push [:MOD, "%"]
      when m = scanner.scan(/\(/)
        @q.push [:LPAREN, "("]
      when m = scanner.scan(/\)/)
        @q.push [:RPAREN, ")"]
      when m = scanner.scan(/\,/)
        @q.push [:COMMA, ","]
      when m = scanner.scan(/\./)
        @q.push [:DOT, "."]
      when m = scanner.scan(/\:\:/)
        @q.push [:DCOLON, "::"]
      when m = scanner.scan(/\:/)
        @q.push [:COLON, ":"]
      when m = scanner.scan(Expression::INSTANCE_VARIABLE)
        @q.push [:INSTANCE_VARIABLE, m[1..-1]]
      when m = scanner.scan(/.|\n/)
        @q.push [m, m]
    end
  end

  @q.push [false, false]

  @q
end