Class: NumberMuncher::Tokenizer

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
lib/number_muncher/tokenizer.rb

Constant Summary collapse

INVALID_REGEX = 
/(?![#{Unicode::VALUES.join}])\S+/.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(string) ⇒ Tokenizer

Returns a new instance of Tokenizer.



11
12
13
 
# File 'lib/number_muncher/tokenizer.rb', line 11

def initialize(string)
  @string = string
end

Instance Attribute Details

#string ⇒ Object (readonly)

Returns the value of attribute string.



9
10
11
 
# File 'lib/number_muncher/tokenizer.rb', line 9

def string
  @string
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



9
10
11
 
# File 'lib/number_muncher/tokenizer.rb', line 9

def tokens
  @tokens
end

Instance Method Details

#call(raise: false) ⇒ Object 



15
16
17
18
19
 
# File 'lib/number_muncher/tokenizer.rb', line 15

def call(raise: false)
  @tokens = []
  scan(raise) until scanner.eos?
  tokens
end