Module: CSVPlusPlus::Lexer

Defined in:
lib/csv_plus_plus/lexer/lexer.rb,
lib/csv_plus_plus/lexer.rb,
lib/csv_plus_plus/lexer/tokenizer.rb

Overview

Common methods to be mixed into the Racc parsers

Defined Under Namespace

Classes: Tokenizer

Constant Summary collapse

END_OF_CODE_SECTION = 
'---'
VARIABLE_REF = 
'$$'
TOKEN_LIBRARY = 
{
  A1_NOTATION: [::CSVPlusPlus::Entities::CellReference::A1_NOTATION_REGEXP, :A1_NOTATION],
  FALSE: [/false/i, :FALSE],
  HEX_COLOR: [::CSVPlusPlus::Color::HEX_STRING_REGEXP, :HEX_COLOR],
  ID: [/[$!\w:]+/, :ID],
  INFIX_OP: [%r{\^|\+|-|\*|/|&|<|>|<=|>=|<>}, :INFIX_OP],
  NUMBER: [/-?[\d.]+/, :NUMBER],
  STRING: [%r{"(?:[^"\\]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*"}, :STRING],
  TRUE: [/true/i, :TRUE],
  VAR_REF: [/\$\$/, :VAR_REF]
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



10
11
12
 
# File 'lib/csv_plus_plus/lexer/lexer.rb', line 10

def tokens
  @tokens
end

Instance Method Details

#initialize(tokens: []) ⇒ Object

Initialize a lexer instance with an empty @tokens



13
14
15
 
# File 'lib/csv_plus_plus/lexer/lexer.rb', line 13

def initialize(tokens: [])
  @tokens = tokens
end

#next_token ⇒ Array<(String, String)>

Used by racc to iterate each token

Returns:

  • (Array<(String, String)>) 


20
21
22
 
# File 'lib/csv_plus_plus/lexer/lexer.rb', line 20

def next_token
  @tokens.shift
end

#parse(input, runtime) ⇒ Lexer#return_value

Orchestate the tokenizing, parsing and error handling of parsing input. Each instance will implement their own

#tokenizer method

Returns:

  • (Lexer#return_value) —

    Each instance will define it’s own return_value with the result of parsing



28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/csv_plus_plus/lexer/lexer.rb', line 28

def parse(input, runtime)
  return if input.nil?

  return return_value unless anything_to_parse?(input)

  tokenize(input, runtime)
  do_parse
  return_value
rescue ::Racc::ParseError => e
  runtime.raise_formula_syntax_error("Error parsing #{parse_subject}", e.message, wrapped_error: e)
rescue ::CSVPlusPlus::Error::ModifierValidationError => e
  raise(::CSVPlusPlus::Error::ModifierSyntaxError.new(runtime, wrapped_error: e))
end