Class: Yay::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/yay/lexer.rb,
lib/yay/lexer_regex.rb

Constant Summary collapse

BASE_PATTERNS =

the expressions to match to find tokens ensure the labels match up to tokens in grammar.y

[
  [:whitespace     , /\s+/],
  [:comment        , /#.*$/],

  # strings
  [:double_quoted  , /"[^"\\]*(?:\\.[^"\\]*)*"/],
  [:single_quoted  , /'[^'\\]*(?:\\.[^'\\]*)*'/],
  [:regex          , /\/[^\/\\\r\n]*(?:\\.[^\/\\\r\n]*)*\/(?:[a-z]*\b)?/],
  [:variable       , /@\w+/],

  # keywords
  [:line           , /\bline[s]?\b/i],
  [:install        , /\binstall\b/i],
  [:list_installed , /\b(list|installed)\b/i],
  [:include        , /\b(include|use|load)\b/i],
  [:and            , /(\b(and|but)\b)|,/i],
  [:verb           , /\b(is|are|a|an)\b/],

  # everything else matched must be a plain old term
  [:literal        , /\b\S+\b/],
]

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(string = "", context_name = nil) ⇒ Lexer

Returns a new instance of Lexer.



8
9
10
11
12
13
14
15
16
 
# File 'lib/yay/lexer.rb', line 8

def initialize(string="", context_name=nil)
  @position     = 0
  @line         = 1
			@context_name = context_name
  
  # default to an empty string scanner. this provides us an endless number
  # of eofs
  use_string(string)
end

Instance Attribute Details

#context_name ⇒ Object

Returns the value of attribute context_name.



6
7
8
 
# File 'lib/yay/lexer.rb', line 6

def context_name
  @context_name
end

Instance Method Details

#get_patterns ⇒ Object 



30
31
32
33
34
35
36
 
# File 'lib/yay/lexer_regex.rb', line 30

def get_patterns
  patterns = BASE_PATTERNS
  # add the colour keywords. generate these from the colour wheel's constants
  colours = Yay::ColourWheel::all_names.join('|')
  patterns.unshift [:colour, Regexp.new("\\b(#{colours})\\b", Regexp::IGNORECASE)]
  return patterns
end

#line ⇒ Object 



31
32
33
 
# File 'lib/yay/lexer.rb', line 31

def line
  @line
end

#next_token ⇒ Object

get the next token in the file



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/yay/lexer.rb', line 36

def next_token
  return nil if @scanner.empty?

  unless @scanner.empty?
    get_patterns.each { |token| 
      type  = token[0]
      value = @scanner.scan(token[1])
      next unless value
      return next_token if type == :whitespace
      return next_token if type == :comment
      @position += 1
      return normalize_token type, value
    }
  end
  return [:junk, @scanner.scan_until(/$/)]
end

#normalize_token(type, value) ⇒ Object

perform transformations on the values



54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/yay/lexer.rb', line 54

def normalize_token type, value  
  case type
  when nil
  when :double_quoted
    value = value[1, value.length - 2]
    while value.sub!("\\\"", "\"") != nil
    end
    type = :literal
  end
  return [type, value]
end

#position ⇒ Object 



27
28
29
 
# File 'lib/yay/lexer.rb', line 27

def position
  @position
end

#use_string(string) ⇒ Object

take a string and begin scanning it



23
24
25
 
# File 'lib/yay/lexer.rb', line 23

def use_string(string)
  @scanner = StringScanner.new string
end