Class: Dendroid::Utils::BaseTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/dendroid/utils/base_tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(&aBlock) ⇒ BaseTokenizer

Returns a new instance of BaseTokenizer.



15
16
17
18
19
20
21
22
23
24
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 15

def initialize(&aBlock)
  @scanner = StringScanner.new('')
  @actions = { skip: [], scan_verbatim: [], scan_value: [] }
  defaults

  if block_given?
    instance_exec(&aBlock)
    # grammar_complete!
  end
end

Instance Attribute Details

#actions ⇒ Object (readonly)

Returns the value of attribute actions.



13
14
15
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 13

def actions
  @actions
end

#line_start ⇒ Object (readonly)

Returns the value of attribute line_start.



12
13
14
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 12

def line_start
  @line_start
end

#lineno ⇒ Object (readonly)

Returns the value of attribute lineno.



11
12
13
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 11

def lineno
  @lineno
end

#scanner ⇒ Object (readonly)

Returns the value of attribute scanner.



10
11
12
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 10

def scanner
  @scanner
end

Instance Method Details

#input=(source) ⇒ Object 



26
27
28
29
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 26

def input=(source)
  scanner.string = source
  reset
end

#map_verbatim2terminal(mapping) ⇒ Object 



75
76
77
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 75

def map_verbatim2terminal(mapping)
  @verbatim2terminal = mapping
end

#next_token ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 79

def next_token
  token = nil

  # Loop until end of input reached or token found
  until scanner.eos?
    if scanner.skip(actions[:skip_nl])
      next_line_scanned
      next
    end

    next if scanner.skip(actions[:skip_ws]) # Skip whitespaces

    if (text = scanner.scan(actions[:scan_verbatim]))
      token = verbatim_scanned(text)
      break
    end

    tuple = actions[:scan_value].find do |(pattern, terminal, conversion)|
      scanner.check(pattern)
    end
    if tuple
      (pattern, terminal, conversion) = tuple
      text = scanner.scan(pattern)
      token = value_scanned(text, terminal, conversion)
      break
    end

    # Unknown token
    col = scanner.pos - line_start + 1
    erroneous = scanner.peek(1).nil? ? '' : scanner.scan(/./)
    raise Exception, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
  end

  token
end

#reset ⇒ Object 



31
32
33
34
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 31

def reset
  @lineno = 1
  @line_start = 0
end

#scan_value(pattern, terminal, convertion) ⇒ Object 



65
66
67
68
69
70
71
72
73
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 65

def scan_value(pattern, terminal, convertion)
  patt = normalize_pattern(pattern)
  tuple = [patt, terminal, convertion]
  if actions[:scan_value].empty?
    actions[:scan_value] = [tuple]
  else
    actions[:scan_verbatim] << tuple
  end
end

#scan_verbatim(pattern) ⇒ Object 



55
56
57
58
59
60
61
62
63
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 55

def scan_verbatim(pattern)
  patt = normalize_pattern(pattern)
  if actions[:scan_verbatim].empty?
    actions[:scan_verbatim] = patt
  else
    new_pattern = actions[:scan_verbatim].union(patt)
    actions[:scan_verbatim] = new_pattern
  end
end

#skip(pattern) ⇒ Object 



46
47
48
49
50
51
52
53
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 46

def skip(pattern)
  if actions[:skip].empty?
    actions[:skip] = pattern
  else
    new_pattern = actions[:skip].union(pattern)
    actions[:skip] = new_pattern
  end
end

#skip_nl(pattern) ⇒ Object

action, pattern, terminal?, conversion? action = skip, skip_nl, scan



38
39
40
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 38

def skip_nl(pattern)
  actions[:skip_nl] = pattern
end

#skip_ws(pattern) ⇒ Object 



42
43
44
 
# File 'lib/dendroid/utils/base_tokenizer.rb', line 42

def skip_ws(pattern)
  actions[:skip_ws] = pattern
end