Class: TBMX::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tbmx.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ Tokenizer

Returns a new instance of Tokenizer.



224
225
226
227
 
# File 'lib/tbmx.rb', line 224

def initialize(text)
  @text = text
  tokenize
end

Instance Attribute Details

#text ⇒ Object (readonly)

Returns the value of attribute text.



223
224
225
 
# File 'lib/tbmx.rb', line 223

def text
  @text
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



223
224
225
 
# File 'lib/tbmx.rb', line 223

def tokens
  @tokens
end

Instance Method Details

#tokenize ⇒ Object 



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
 
# File 'lib/tbmx.rb', line 229

def tokenize
  @tokens = []
  rest = text.gsub("\r", "")
  while rest.length > 0
    if result =     BackslashToken.matches?(rest) or # Single Character Tokens
       result =     LeftBraceToken.matches?(rest) or
       result =    RightBraceToken.matches?(rest) or
       result = EmptyNewlinesToken.matches?(rest) or # String Tokens
       result =    WhitespaceToken.matches?(rest) or
       result =        NumberToken.matches?(rest) or
       result =          WordToken.matches?(rest)
    then
      @tokens << result[0]
      rest = result[1]
    else
      raise RuntimeError, "Couldn't tokenize the remaining text."
    end
  end
  return @tokens
end