Class: Layo::Tokenizer

Inherits:
Object
  • Object
show all
Includes:
Peekable
Defined in:
lib/layo/tokenizer.rb

Instance Attribute Summary collapse

Attributes included from Peekable

#peek_index

Instance Method Summary collapse

Methods included from Peekable

#next, #peek, #reset, #reset_peek, #unpeek

Constructor Details

#initialize(lexer) ⇒ Tokenizer

Returns a new instance of Tokenizer.



29
30
31
32
33
 
# File 'lib/layo/tokenizer.rb', line 29

def initialize(lexer)
  @lexer = lexer
  init_token_table
  reset
end

Instance Attribute Details

#lexer ⇒ Object

Instance of Layo::Lexer



27
28
29
 
# File 'lib/layo/tokenizer.rb', line 27

def lexer
  @lexer
end

Instance Method Details

#init_token_table ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/layo/tokenizer.rb', line 52

def init_token_table
  @token_table = { :list => {} }
  ['HAI', 'KTHXBYE', 'NOOB', 'TROOF', 'NUMBR', 'NUMBAR',
    'YARN', 'I HAS A', 'ITZ', 'R', 'SUM OF', 'DIFF OF', 'PRODUKT OF',
    'QUOSHUNT OF', 'MOD OF', 'BIGGR OF', 'SMALLR OF', 'BOTH OF', 'EITHER OF',
    'WON OF', 'NOT', 'ALL OF', 'ANY OF', 'BOTH SAEM', 'DIFFRINT',
    'SMOOSH', 'MAEK', 'IS NOW A', 'A', 'VISIBLE', 'GIMMEH', 'MKAY', 'AN',
    'O RLY?', 'YA RLY', 'NO WAI', 'MEBBE', 'OIC', 'WTF?', 'OMG', 'OMGWTF',
    'GTFO', 'IM IN YR', 'YR', 'TIL', 'WILE', 'IM OUTTA YR', 'UPPIN',
    'NERFIN', 'HOW DUZ I', 'AN YR', 'IF U SAY SO', 'FOUND YR'
  ].each do |t|
    lexemes = t.split(' ')
    root = @token_table[:list]
    lexemes[0..-2].each do |lexeme|
      root[lexeme] = {} unless root.has_key?(lexeme)
      root[lexeme][:list] = {} unless root[lexeme].has_key?(:list)
      root = root[lexeme][:list]
    end
    root[lexemes.last] = { match: t.gsub(' ', '_').downcase.to_sym }
  end
end

#match_longest(lexeme, root) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/layo/tokenizer.rb', line 74

def match_longest(lexeme, root)
  return nil unless root.has_key?(:list) && root[:list].has_key?(lexeme)
  newroot = root[:list][lexeme]
  best_match = newroot.has_key?(:match) ? newroot[:match] : nil
  next_lexeme = @lexer.peek
  unless next_lexeme.nil?
    try_match = match_longest(next_lexeme[0], newroot)
    best_match = try_match unless try_match.nil?
  end
  best_match
end

#next_item ⇒ Object

Tries to convert next lexeme in lexeme stream into token and return it

Raises:



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/layo/tokenizer.rb', line 87

def next_item
  lexeme, token = @lexer.next, nil
  if lexeme[0].nil?
    token = { type: :eof }
  elsif lexeme[0].lol_string?
    token = { type: :string, data: lexeme[0][1..-2] }
  elsif lexeme[0].lol_integer?
    token = { type: :integer, data: lexeme[0].to_i }
  elsif lexeme[0].lol_float?
    token = { type: :float, data: lexeme[0].to_f }
  elsif lexeme[0].lol_boolean?
    token = { type: :boolean, data: (lexeme[0] == 'WIN') }
  elsif lexeme[0] == '!'
    token = { type: :exclamation }
  elsif lexeme[0] == "\n"
    token = { type: :newline }
  else
    # Try to match keyword
    token_type = match_longest(lexeme[0], @token_table)
    unless token_type.nil?
      token = { type: token_type }
      # Consume all peeked lexemes
      token_type.to_s.count('_').times { @lexer.next }
    else
      # Try to match identifier
      if lexeme[0].lol_identifier?
        token = { type: :identifier, data: lexeme[0] }
      end
    end
  end
  raise UnknownTokenError.new(lexeme) if token.nil?
  token.merge(line: lexeme[1], pos: lexeme[2])
end

#try(*types) ⇒ Object

Checks that following n tokens have the same types as in given ‘types’ array. Each element of ‘types’ can be either a symbol or array of symbols This method does not modify peek index



38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/layo/tokenizer.rb', line 38

def try(*types)
  index = @peek_index
  result = true
  types.each do |type|
    allowed = type.is_a?(Array) ? type : [type]
    unless allowed.include?(peek[:type])
      result = false
      break
    end
  end
  @peek_index = index
  result
end