Class: Antlr4::Runtime::Lexer

Inherits:
Recognizer show all
Defined in:
lib/antlr4/runtime/lexer.rb

Constant Summary collapse

DEFAULT_MODE = 
0
MORE = 
-2
SKIP = 
-3
DEFAULT_TOKEN_CHANNEL = 
Token::DEFAULT_CHANNEL
HIDDEN = 
Token::HIDDEN_CHANNEL
MIN_CHAR_VALUE = 
0x0000
MAX_CHAR_VALUE = 
0x10FFFF

Constants inherited from Recognizer

Recognizer::EOF

Instance Attribute Summary collapse

Attributes inherited from Recognizer

#_interp, #_state_number

Instance Method Summary collapse

Methods inherited from Recognizer

#action, #add_error_listener, #error_header, #error_listener_dispatch, #get_rule_index_map, #get_serialized_atn, #get_token_type, #get_token_type_map, #get_vocabulary, #parse_info, #precpred, #remove_error_listener, #remove_error_listeners, #rule_names, #sempred, #token_error_display, #token_names

Constructor Details

#initialize(input = nil) ⇒ Lexer

Returns a new instance of Lexer.



45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/antlr4/runtime/lexer.rb', line 45

def initialize(input = nil)
  super()
  unless input.nil?
    @_input = input
    @_token_factory_source_pair = OpenStruct.new
    @_token_factory_source_pair.a = self
    @_token_factory_source_pair.b = input
  end
  @_mode_stack = []
  reset
  @_factory = CommonTokenFactory.instance
end

Instance Attribute Details

#_channel ⇒ Object

Returns the value of attribute _channel.



19
20
21
 
# File 'lib/antlr4/runtime/lexer.rb', line 19

def _channel
  @_channel
end

#_hit_eof ⇒ Object

Returns the value of attribute _hit_eof.



18
19
20
 
# File 'lib/antlr4/runtime/lexer.rb', line 18

def _hit_eof
  @_hit_eof
end

#_input ⇒ Object

Returns the value of attribute _input.



13
14
15
 
# File 'lib/antlr4/runtime/lexer.rb', line 13

def _input
  @_input
end

#_mode ⇒ Object

Returns the value of attribute _mode.



22
23
24
 
# File 'lib/antlr4/runtime/lexer.rb', line 22

def _mode
  @_mode
end

#_mode_stack ⇒ Object

Returns the value of attribute _mode_stack.



21
22
23
 
# File 'lib/antlr4/runtime/lexer.rb', line 21

def _mode_stack
  @_mode_stack
end

#_text ⇒ Object

Returns the value of attribute _text.



23
24
25
 
# File 'lib/antlr4/runtime/lexer.rb', line 23

def _text
  @_text
end

#_token_start_char_index ⇒ Object

Returns the value of attribute _token_start_char_index.



15
16
17
 
# File 'lib/antlr4/runtime/lexer.rb', line 15

def _token_start_char_index
  @_token_start_char_index
end

#_token_start_char_position_in_line ⇒ Object

Returns the value of attribute _token_start_char_position_in_line.



17
18
19
 
# File 'lib/antlr4/runtime/lexer.rb', line 17

def _token_start_char_position_in_line
  @_token_start_char_position_in_line
end

#_token_start_line ⇒ Object

Returns the value of attribute _token_start_line.



16
17
18
 
# File 'lib/antlr4/runtime/lexer.rb', line 16

def _token_start_line
  @_token_start_line
end

#_type ⇒ Object

Returns the value of attribute _type.



20
21
22
 
# File 'lib/antlr4/runtime/lexer.rb', line 20

def _type
  @_type
end

#token ⇒ Object

Returns the value of attribute token.



14
15
16
 
# File 'lib/antlr4/runtime/lexer.rb', line 14

def token
  @token
end

Instance Method Details

#all_tokens ⇒ Object 



200
201
202
203
204
205
206
207
208
 
# File 'lib/antlr4/runtime/lexer.rb', line 200

def all_tokens
  tokens = []
  t = next_token
  while t.type != Token::EOF
    tokens << t
    t = next_token
  end
  tokens
end

#char_error_display(c) ⇒ Object 



251
252
253
254
 
# File 'lib/antlr4/runtime/lexer.rb', line 251

def char_error_display(c)
  s = error_display_char(c)
  "'" + s + "'"
end

#char_index ⇒ Object 



190
191
192
 
# File 'lib/antlr4/runtime/lexer.rb', line 190

def char_index
  @_input.index
end

#char_position_in_line ⇒ Object 



178
179
180
 
# File 'lib/antlr4/runtime/lexer.rb', line 178

def char_position_in_line
  @_interp.char_position_in_line
end

#emit(token = nil) ⇒ Object 



159
160
161
162
163
164
165
 
# File 'lib/antlr4/runtime/lexer.rb', line 159

def emit(token = nil)
  if !token.nil?
    @_token = token
  else
    @_token = @_factory.create(@_token_factory_source_pair, @_type, @_text, @_channel, @_token_start_char_index, char_index - 1, @_token_start_line, @_token_start_char_position_in_line)
  end
end

#emit_eof ⇒ Object 



167
168
169
170
171
172
 
# File 'lib/antlr4/runtime/lexer.rb', line 167

def emit_eof
  cpos = char_position_in_line
  eof = @_factory.create(@_token_factory_source_pair, Token::EOF, nil, Token::DEFAULT_CHANNEL, @_input.index, @_input.index - 1, line, cpos)
  emit(eof)
  eof
end

#error_display(s) ⇒ Object 



225
226
227
228
229
230
231
 
# File 'lib/antlr4/runtime/lexer.rb', line 225

def error_display(s)
  buf = ''
  s.chars.each do |c|
    buf << error_display_char(c)
  end
  buf
end

#error_display_char(c) ⇒ Object 



233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/antlr4/runtime/lexer.rb', line 233

def error_display_char(c)
  s = ''
  s << c
  case c
  when Token::EOF
    s = '<EOF>'
  when '\n'
    s = '\\n'
  when '\t'
    s = '\\t'
  when '\r'
    s = '\\r'
  else
    # type code here
  end
  s
end

#input_stream(input) ⇒ Object 



144
145
146
147
148
149
150
151
152
153
 
# File 'lib/antlr4/runtime/lexer.rb', line 144

def input_stream(input)
  @_input = nil
  @_token_factory_source_pair = OpenStruct.new
  @_token_factory_source_pair.a = self
  @_token_factory_source_pair.b = @_input
  reset
  @_input = input
  @_token_factory_source_pair.a = self
  @_token_factory_source_pair.b = @_input
end

#line ⇒ Object 



174
175
176
 
# File 'lib/antlr4/runtime/lexer.rb', line 174

def line
  @_interp.line
end

#mode(m) ⇒ Object 



126
127
128
 
# File 'lib/antlr4/runtime/lexer.rb', line 126

def mode(m)
  @_mode = m
end

#more ⇒ Object 



122
123
124
 
# File 'lib/antlr4/runtime/lexer.rb', line 122

def more
  @_type = MORE
end

#next_token ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/antlr4/runtime/lexer.rb', line 66

def next_token
  if @_input.nil?
    raise IllegalStateException, 'next_token requires a non-nil input stream.'
  end

  # Mark start location in char stream so unbuffered streams are
  # guaranteed at least have text of current token
  token_start_marker = @_input.mark
  begin
    repeat_outer = true
    repeat_outer = next_token_inner while repeat_outer
    return @_token
  ensure # make sure we release marker after match or
    # unbuffered char stream will keep buffering
    @_input.release(token_start_marker)
  end
end

#next_token_inner ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/antlr4/runtime/lexer.rb', line 84

def next_token_inner
  loop do
    if @_hit_eof
      emit_eof
      return false
    end

    @_token = nil
    @_channel = Token::DEFAULT_CHANNEL
    @_token_start_char_index = @_input.index
    @_token_start_char_position_in_line = @_interp.char_position_in_line
    @_token_start_line = @_interp.line
    @_text = nil
    loop do
      @_type = Token::INVALID_TYPE

      begin
        ttype = @_interp.match(@_input, @_mode)
      rescue LexerNoViableAltException => e
        notify_listeners(e) # report error
        recover1(e)
        ttype = SKIP
      end
      @_hit_eof = true if @_input.la(1) == IntStream::EOF
      @_type = ttype if @_type == Token::INVALID_TYPE
      return true if @_type == SKIP
      break if @_type != MORE
    end

    emit if @_token.nil?
    return false
  end
end

#notify_listeners(e) ⇒ Object 



217
218
219
220
221
222
223
 
# File 'lib/antlr4/runtime/lexer.rb', line 217

def notify_listeners(e)
  text = @_input.text(Interval.of(@_token_start_char_index, @_input.index))
  msg = "token recognition error at: '" + error_display(text) + "'"

  listener = error_listener_dispatch
  listener.syntax_error(self, nil, @_token_start_line, @_token_start_char_position_in_line, msg, e)
end

#pop_mode ⇒ Object

Raises:

  • (EmptyStackException) 


136
137
138
139
140
141
142
 
# File 'lib/antlr4/runtime/lexer.rb', line 136

def pop_mode
  raise EmptyStackException if @_mode_stack.empty?

  puts('popMode back to ' + @_mode_stack[-1]) if LexerATNSimulator.debug
  mode(@_mode_stack.pop)
  @_mode
end

#push_mode(m) ⇒ Object 



130
131
132
133
134
 
# File 'lib/antlr4/runtime/lexer.rb', line 130

def push_mode(m)
  puts('pushMode ' + m) if LexerATNSimulator.debug
  @_mode_stack.push(@_mode)
  mode(m)
end

#recover1(_e) ⇒ Object 



210
211
212
213
214
215
 
# File 'lib/antlr4/runtime/lexer.rb', line 210

def recover1(_e)
  if @_input.la(1) != IntStream::EOF
    # skip a char and begin again
    @_interp.consume(@_input)
  end
end

#recover2(_re) ⇒ Object 



256
257
258
 
# File 'lib/antlr4/runtime/lexer.rb', line 256

def recover2(_re)
  @_input.consume
end

#reset ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/antlr4/runtime/lexer.rb', line 25

def reset
  # wack Lexer state variables
  unless @_input.nil?
    @_input.seek(0) # rewind the input
  end
  @_token = nil
  @_type = Token::INVALID_TYPE
  @_channel = Token::DEFAULT_CHANNEL
  @_token_start_char_index = -1
  @_token_start_char_position_in_line = -1
  @_token_start_line = -1
  @_text = nil

  @_hit_eof = false
  @_mode = DEFAULT_MODE
  @_mode_stack.clear

  @_interp.reset unless @_interp.nil?
end

#set_char_position_in_line(char_position_in_line) ⇒ Object 



186
187
188
 
# File 'lib/antlr4/runtime/lexer.rb', line 186

def set_char_position_in_line(char_position_in_line)
  @_interp.set_char_position_in_line(char_position_in_line)
end

#set_line(line) ⇒ Object 



182
183
184
 
# File 'lib/antlr4/runtime/lexer.rb', line 182

def set_line(line)
  @_interp.set_line(line)
end

#skip ⇒ Object 



118
119
120
 
# File 'lib/antlr4/runtime/lexer.rb', line 118

def skip
  @_type = SKIP
end

#source_name ⇒ Object 



155
156
157
 
# File 'lib/antlr4/runtime/lexer.rb', line 155

def source_name
  @_input.get_source_name
end

#text ⇒ Object 



194
195
196
197
198
 
# File 'lib/antlr4/runtime/lexer.rb', line 194

def text
  return @_text unless @_text.nil?

  @_interp.text(@_input)
end

#token_factory ⇒ Object 



58
59
60
 
# File 'lib/antlr4/runtime/lexer.rb', line 58

def token_factory
  @_factory
end

#token_factory=(factory) ⇒ Object 



62
63
64
 
# File 'lib/antlr4/runtime/lexer.rb', line 62

def token_factory=(factory)
  @_factory = factory
end