Class: BaseTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/support/base_tokenizer.rb

Defined Under Namespace

Classes: ScanError

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source) ⇒ BaseTokenizer

Constructor. Initialize a tokenizer for Skeem.

Parameters:

  • source (String) —

    Skeem text to tokenize.



15
16
17
18
 
# File 'lib/support/base_tokenizer.rb', line 15

def initialize(source)
  @scanner = StringScanner.new('')
  restart(source)
end

Instance Attribute Details

#line_start ⇒ Object (readonly)

Returns the value of attribute line_start.



9
10
11
 
# File 'lib/support/base_tokenizer.rb', line 9

def line_start
  @line_start
end

#lineno ⇒ Object (readonly)

Returns the value of attribute lineno.



8
9
10
 
# File 'lib/support/base_tokenizer.rb', line 8

def lineno
  @lineno
end

#scanner ⇒ Object (readonly)

Returns the value of attribute scanner.



7
8
9
 
# File 'lib/support/base_tokenizer.rb', line 7

def scanner
  @scanner
end

Instance Method Details

#restart(source) ⇒ Object

Parameters:

  • source (String) —

    Skeem text to tokenize.



21
22
23
24
25
 
# File 'lib/support/base_tokenizer.rb', line 21

def restart(source)
  @scanner.string = source
  @lineno = 1
  @line_start = 0
end

#tokens ⇒ Array<SkmToken>

Returns | Returns a sequence of tokens.

Returns:

  • (Array<SkmToken>) —

    | Returns a sequence of tokens



28
29
30
31
32
33
34
35
36
 
# File 'lib/support/base_tokenizer.rb', line 28

def tokens
  tok_sequence = []
  until @scanner.eos?
    token = _next_token
    tok_sequence << token unless token.nil?
  end

  return tok_sequence
end