Module: Scanner

Includes:
Enumerable
Defined in:
lib/scanner/scanner.rb,
lib/scanner.rb,
lib/scanner/token.rb,
lib/scanner/version.rb

Overview

THis is the module to include in your class to inherit the scanner functionality

Defined Under Namespace

Classes: Token

Constant Summary collapse

VERSION = 
"0.0.4"

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.append_features(aModule) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/scanner/scanner.rb', line 4

def self.append_features(aModule)
  super

  aModule.instance_eval do
    @language_tokens = {}
    @ignore = nil
    @keywords = nil
    @check_for_token_separator = {}
    @separator = nil

    # defines a token of the language
    # Each token is defined by a symbol, used to identify the token, and a
    # regular expression that the token should match. An optional third
    # parameter accepts a hash of options. For example:
    # 
    #     token :number, '\d+'
    # 
    # will match strings containing digits.
    # 
    # Some care is needed when defining tokens that collide with other
    # tokens. For instance, a languange may define the token '==' and the
    # token '='. You need to define the double equals before the single
    # equals, otherwise the string '==' will be identified as two '=' tokens,
    # instead of a '==' token. 
    def token(token_symbol, regular_expression, options = {})
      modified_reg_exp = "\\A#{regular_expression}"
      @language_tokens[token_symbol] = /#{modified_reg_exp}/
      @check_for_token_separator[token_symbol] = options[:check_for_token_separator] == true
    end

    def ignore(regular_expression)
      modified_reg_exp = "\\A#{regular_expression}"
      @ignore = /#{modified_reg_exp}/
    end

    def keywords(keywords)
      @keywords = keywords
    end

    def token_separator(regular_expression)
      modified_reg_exp = "\\A#{regular_expression}"
      @separator = /#{modified_reg_exp}/
    end

    token :eof, '\z'
  end

end

Instance Method Details

#consume ⇒ Object 



87
88
89
90
91
92
93
94
 
# File 'lib/scanner/scanner.rb', line 87

def consume
  if @token_number >= @token_list.size
    consume_next_token
  end
  token = @token_list[@token_number]
  @token_number+=1
  token
end

#each ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/scanner/scanner.rb', line 120

def each
  local_index = 0
  begin
    if local_index >= @token_list.size
      consume_next_token
    end
    current_token = @token_list[local_index]
    if current_token.is_not? :eof
      yield current_token
    end
    local_index += 1
  end while current_token.is_not? :eof
end

#look_ahead(number_of_tokens = 1) ⇒ Object 



96
97
98
99
100
101
 
# File 'lib/scanner/scanner.rb', line 96

def look_ahead(number_of_tokens = 1)
  while @token_list.size < @token_number + number_of_tokens
    consume_next_token
  end
  @token_list[@token_number + number_of_tokens - 1]
end

#parse(program) ⇒ Object

Sets the string to parse



79
80
81
82
83
84
85
 
# File 'lib/scanner/scanner.rb', line 79

def parse(program)
  @program = program
  @token_list = []
  @line_number = 1
  @column_number = 1
  @token_number = 0
end

#token_is?(token_type) ⇒ Boolean

Returns:

  • (Boolean) 


103
104
105
 
# File 'lib/scanner/scanner.rb', line 103

def token_is?(token_type)
  look_ahead.is? token_type
end

#token_is_not?(token_type) ⇒ Boolean

Returns:

  • (Boolean) 


107
108
109
 
# File 'lib/scanner/scanner.rb', line 107

def token_is_not?(token_type)
  not (look_ahead.is? token_type)
end

#tokens_are?(*tokens) ⇒ Boolean

Returns:

  • (Boolean) 


111
112
113
114
115
116
117
118
 
# File 'lib/scanner/scanner.rb', line 111

def tokens_are?(*tokens)
  look_ahead_index = 1
  tokens.each do |token|
    return false unless look_ahead(look_ahead_index).is? token
    look_ahead_index += 1
  end
  return true
end