Class: Keisan::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/keisan/tokenizer.rb

Constant Summary collapse

TOKEN_CLASSES = 
[
  Tokens::Group,
  Tokens::String,
  Tokens::Null,
  Tokens::Boolean,
  Tokens::Word,
  Tokens::Number,
  Tokens::Assignment,
  Tokens::LogicalOperator,
  Tokens::ArithmeticOperator,
  Tokens::BitwiseOperator,
  Tokens::Comma,
  Tokens::Colon,
  Tokens::Dot,
  Tokens::LineSeparator,
  Tokens::Unknown
]
TOKEN_REGEX = 
Regexp::new(
  TOKEN_CLASSES.map(&:regex).join("|")
)

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(expression) ⇒ Tokenizer

Returns a new instance of Tokenizer.



27
28
29
30
31
 
# File 'lib/keisan/tokenizer.rb', line 27

def initialize(expression)
  @expression = self.class.normalize_expression(expression)
  @scan = @expression.scan(TOKEN_REGEX)
  @tokens = tokenize!
end

Instance Attribute Details

#expression ⇒ Object (readonly)

Returns the value of attribute expression.



25
26
27
 
# File 'lib/keisan/tokenizer.rb', line 25

def expression
  @expression
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



25
26
27
 
# File 'lib/keisan/tokenizer.rb', line 25

def tokens
  @tokens
end

Class Method Details

.normalize_expression(expression) ⇒ Object 



33
34
35
36
 
# File 'lib/keisan/tokenizer.rb', line 33

def self.normalize_expression(expression)
  expression = normalize_line_delimiters(expression)
  expression = remove_comments(expression)
end