Class: Bijou::Parse::Lexer
- Inherits:
-
Object
- Object
- Bijou::Parse::Lexer
show all
- Defined in:
- lib/bijou/lexer.rb
Overview
The base class shared by the lexers. It encapsulates common functionality, such as token management and error reporting.
Instance Attribute Summary collapse
Instance Method Summary
collapse
-
#column ⇒ Object
The current column number being scanned, used for diagnostics.
-
#diagnostic(m, l, c) ⇒ Object
-
#error(s) ⇒ Object
-
#errors ⇒ Object
-
#getc ⇒ Object
Returns the next character from the input stream.
-
#initialize(input) ⇒ Lexer
constructor
-
#is_token_buffered ⇒ Object
-
#line ⇒ Object
The current line number being scanned, used for diagnostics.
-
#next_token ⇒ Object
-
#peek_token ⇒ Object
-
#pop ⇒ Object
Used for backtracking, puts the most recently removed character back into the input stream.
-
#pop_token ⇒ Object
Used for backtracking by the one token.
-
#prev_token ⇒ Object
-
#push_token(token, text) ⇒ Object
-
#set_string_token(startToken, double) ⇒ Object
-
#shift_token ⇒ Object
Used after a pop operation.
-
#text ⇒ Object
-
#token ⇒ Object
-
#ungetc(ch) ⇒ Object
Used for backtracking, puts the most recently removed character back into the input stream.
-
#warning(s, l = nil, c = nil) ⇒ Object
-
#warnings ⇒ Object
Constructor Details
#initialize(input) ⇒ Lexer
Returns a new instance of Lexer.
119
120
121
122
123
124
125
126
127
128
|
# File 'lib/bijou/lexer.rb', line 119
def initialize(input)
@input = input
@token = nil
@text = ''
@prev_token = nil
@prev_text = ''
@next_token = nil
@next_text = ''
end
|
Instance Attribute Details
Returns the value of attribute input.
117
118
119
|
# File 'lib/bijou/lexer.rb', line 117
def input
@input
end
|
Instance Method Details
#column ⇒ Object
The current column number being scanned, used for diagnostics.
144
145
146
|
# File 'lib/bijou/lexer.rb', line 144
def column
@input.column
end
|
#diagnostic(m, l, c) ⇒ Object
166
167
168
169
170
171
|
# File 'lib/bijou/lexer.rb', line 166
def diagnostic(m, l, c)
if !l; l = line; end
if !c; c = column; end
m.at(l, c)
m
end
|
#error(s) ⇒ Object
179
180
181
182
183
|
# File 'lib/bijou/lexer.rb', line 179
def error(s, l=nil, c=nil)
m = Bijou::Parse::Error.new
m << s
@input.diagnostics.add_error(diagnostic(m, l, c))
end
|
#errors ⇒ Object
196
197
198
|
# File 'lib/bijou/lexer.rb', line 196
def errors()
@input.diagnostics.errors
end
|
#getc ⇒ Object
Returns the next character from the input stream.
149
150
151
|
# File 'lib/bijou/lexer.rb', line 149
def getc()
@input.getc
end
|
#is_token_buffered ⇒ Object
234
235
236
|
# File 'lib/bijou/lexer.rb', line 234
def is_token_buffered
@next_token ? true : false
end
|
#line ⇒ Object
The current line number being scanned, used for diagnostics.
139
140
141
|
# File 'lib/bijou/lexer.rb', line 139
def line
@input.line
end
|
#next_token ⇒ Object
238
239
240
241
242
243
244
|
# File 'lib/bijou/lexer.rb', line 238
def next_token
if is_token_buffered
shift_token
else
nil
end
end
|
#peek_token ⇒ Object
230
231
232
|
# File 'lib/bijou/lexer.rb', line 230
def peek_token
@next_token
end
|
#pop ⇒ Object
Used for backtracking, puts the most recently removed character back into the input stream. The character was automatically buffered when getc was called.
162
163
164
|
# File 'lib/bijou/lexer.rb', line 162
def pop()
@input.pop
end
|
#pop_token ⇒ Object
Used for backtracking by the one token
211
212
213
214
215
216
217
218
219
|
# File 'lib/bijou/lexer.rb', line 211
def pop_token
@next_token = @token
@next_text = @text
@token = @prev_token
@text = @prev_text
@prev_token = nil
@prev_text = ''
@token
end
|
#prev_token ⇒ Object
226
227
228
|
# File 'lib/bijou/lexer.rb', line 226
def prev_token
@prev_token
end
|
#push_token(token, text) ⇒ Object
200
201
202
203
204
205
206
207
208
|
# File 'lib/bijou/lexer.rb', line 200
def push_token(token, text)
@prev_token = @token
@prev_text = @text
@token = token
@text = text
@next_token = nil
@next_text = ''
@token
end
|
#set_string_token(startToken, double) ⇒ Object
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
|
# File 'lib/bijou/lexer.rb', line 246
def set_string_token(startToken, double)
start_line = @line
start_column = @column
buf = startToken
while ch0 = getc
ch = ch0.chr
buf << ch
if ch == "\\"
ch1 = getc
buf << ch1.chr
elsif ch == '"'
if double
return push_token(Token::String, buf)
end
elsif ch == "'"
if !double
return push_token(Token::String, buf)
end
end
end
warning("unterminated string literal", start_line, start_column)
return nil
end
|
#shift_token ⇒ Object
Used after a pop operation
222
223
224
|
# File 'lib/bijou/lexer.rb', line 222
def shift_token
push_token @next_token, @next_text
end
|
#text ⇒ Object
134
135
136
|
# File 'lib/bijou/lexer.rb', line 134
def text
@text
end
|
#token ⇒ Object
130
131
132
|
# File 'lib/bijou/lexer.rb', line 130
def token
@token
end
|
#ungetc(ch) ⇒ Object
Used for backtracking, puts the most recently removed character back into the input stream.
155
156
157
|
# File 'lib/bijou/lexer.rb', line 155
def ungetc(ch)
@input.ungetc(ch)
end
|
#warning(s, l = nil, c = nil) ⇒ Object
173
174
175
176
177
|
# File 'lib/bijou/lexer.rb', line 173
def warning(s, l=nil, c=nil)
m = Bijou::Parse::Warning.new
m << s
@input.diagnostics.add_warning(diagnostic(m, l, c))
end
|
#warnings ⇒ Object
192
193
194
|
# File 'lib/bijou/lexer.rb', line 192
def warnings()
@input.diagnostics.warnings
end
|