Class: Yay::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/yay/lexer.rb,
lib/yay/lexer_regex.rb

Overview

tokenises yay rules

Constant Summary collapse

BASE_PATTERNS =

the expressions to match to find tokens ensure the labels match up to tokens in grammar.y

[
  [:whitespace     , /\s+/],
  [:comment        , /#.*$/],

  # strings
  [:double_quoted  , /"[^"\\]*(?:\\.[^"\\]*)*"/],
  [:single_quoted  , /'[^'\\]*(?:\\.[^'\\]*)*'/],
  [:regex          , /\/[^\/\\\r\n]*(?:\\.[^\/\\\r\n]*)*\/(?:[a-z]*\b)?/],
  [:variable       , /@\w+/],

  # keywords
  [:line           , /\bline[s]?\b/i],
  [:install        , /\binstall\b/i],
  [:list_installed , /\b(list|installed)\b/i],
  [:include        , /\b(include|use|load)\b/i],
  [:and            , /(\b(and|but)\b)|,/i],
  [:verb           , /\b(is|are|a|an)\b/],

  # everything else matched must be a plain old term
  [:literal        , /\b\S+\b/],
]

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(string = "", context_name = nil) ⇒ Lexer

Returns a new instance of Lexer.



9
10
11
12
13
14
15
16
17
 
# File 'lib/yay/lexer.rb', line 9

def initialize(string="", context_name=nil)
  @position     = 1
  @line         = 1
			@context_name = context_name
  
  # default to an empty string scanner. this provides us an endless number
  # of eofs
  use_string(string)
end

Instance Attribute Details

#context_name ⇒ Object

Returns the value of attribute context_name.



7
8
9
 
# File 'lib/yay/lexer.rb', line 7

def context_name
  @context_name
end

Instance Method Details

#get_patterns ⇒ Object

get the regular expressions we need. always use this instead of referencing the base patterns directly as we augment that array with the colour names that are available



33
34
35
36
37
38
39
 
# File 'lib/yay/lexer_regex.rb', line 33

def get_patterns
  patterns = BASE_PATTERNS
  # add the colour keywords. generate these from the colour wheel's constants
  colours = Yay::ColourWheel::all_names.join('|')
  patterns.unshift [:colour, Regexp.new("\\b(#{colours})\\b", Regexp::IGNORECASE)]
  return patterns
end

#line ⇒ Object

return the current line



36
37
38
 
# File 'lib/yay/lexer.rb', line 36

def line
  @line
end

#next_token ⇒ Object

get the next token in the file



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/yay/lexer.rb', line 41

def next_token
  return nil if @scanner.empty?

  unless @scanner.empty?
    get_patterns.each { |token| 
      type  = token[0]
      value = @scanner.scan(token[1])
      next unless value
      return next_token if type == :whitespace
      return next_token if type == :comment
      @position += 1
      return normalize_token type, value
    }
  end
  return [:junk, @scanner.scan_until(/$/)]
end

#normalize_token(type, value) ⇒ Object

perform transformations on the values



59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/yay/lexer.rb', line 59

def normalize_token type, value  
  case type
  when nil
  when :double_quoted
    value = value[1, value.length - 2]
    while value.sub!("\\\"", "\"") != nil
    end
    type = :literal
  end
  return [type, value]
end

#position ⇒ Object

return the current word



31
32
33
 
# File 'lib/yay/lexer.rb', line 31

def position
  @position
end

#use_string(string) ⇒ Object

take a string and begin scanning it



26
27
28
 
# File 'lib/yay/lexer.rb', line 26

def use_string(string)
  @scanner = StringScanner.new string
end