Class: Bijou::Parse::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/bijou/lexer.rb

Overview

The base class shared by the lexers. It encapsulates common functionality, such as token management and error reporting.

Direct Known Subclasses

StringLexer, TagLexer, TextLexer

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ Lexer

Returns a new instance of Lexer.



119
120
121
122
123
124
125
126
127
128
# File 'lib/bijou/lexer.rb', line 119

def initialize(input)
  @input = input
  @token = nil
  @text = ''
  # One token lookahead for LL(1)
  @prev_token = nil
  @prev_text = ''
  @next_token = nil
  @next_text = ''
end

Instance Attribute Details

#inputObject (readonly)

Returns the value of attribute input.



117
118
119
# File 'lib/bijou/lexer.rb', line 117

def input
  @input
end

Instance Method Details

#columnObject

The current column number being scanned, used for diagnostics.



144
145
146
# File 'lib/bijou/lexer.rb', line 144

def column
  @input.column
end

#diagnostic(m, l, c) ⇒ Object



166
167
168
169
170
171
# File 'lib/bijou/lexer.rb', line 166

def diagnostic(m, l, c)
  if !l; l = line; end
  if !c; c = column; end
  m.at(l, c)
  m
end

#error(s) ⇒ Object



179
180
181
182
183
# File 'lib/bijou/lexer.rb', line 179

def error(s, l=nil, c=nil)
  m = Bijou::Parse::Error.new
  m << s
  @input.diagnostics.add_error(diagnostic(m, l, c))
end

#errorsObject



196
197
198
# File 'lib/bijou/lexer.rb', line 196

def errors()
  @input.diagnostics.errors
end

#getcObject

Returns the next character from the input stream.



149
150
151
# File 'lib/bijou/lexer.rb', line 149

def getc()
  @input.getc
end

#is_token_bufferedObject



234
235
236
# File 'lib/bijou/lexer.rb', line 234

def is_token_buffered
  @next_token ? true : false
end

#lineObject

The current line number being scanned, used for diagnostics.



139
140
141
# File 'lib/bijou/lexer.rb', line 139

def line
  @input.line
end

#next_tokenObject



238
239
240
241
242
243
244
# File 'lib/bijou/lexer.rb', line 238

def next_token
  if is_token_buffered
    shift_token
  else
    nil
  end
end

#peek_tokenObject



230
231
232
# File 'lib/bijou/lexer.rb', line 230

def peek_token
  @next_token
end

#popObject

Used for backtracking, puts the most recently removed character back into the input stream. The character was automatically buffered when getc was called.



162
163
164
# File 'lib/bijou/lexer.rb', line 162

def pop()
  @input.pop
end

#pop_tokenObject

Used for backtracking by the one token



211
212
213
214
215
216
217
218
219
# File 'lib/bijou/lexer.rb', line 211

def pop_token
  @next_token = @token
  @next_text = @text
  @token = @prev_token
  @text = @prev_text
  @prev_token = nil
  @prev_text = ''
  @token
end

#prev_tokenObject



226
227
228
# File 'lib/bijou/lexer.rb', line 226

def prev_token
  @prev_token
end

#push_token(token, text) ⇒ Object



200
201
202
203
204
205
206
207
208
# File 'lib/bijou/lexer.rb', line 200

def push_token(token, text)
  @prev_token = @token
  @prev_text = @text
  @token = token
  @text = text
  @next_token = nil
  @next_text = ''
  @token
end

#set_string_token(startToken, double) ⇒ Object



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
# File 'lib/bijou/lexer.rb', line 246

def set_string_token(startToken, double)
  start_line = @line
  start_column = @column

  buf = startToken # Raw string

  while ch0 = getc
    ch = ch0.chr
    buf << ch

    if ch == "\\"
      # Ignore the next character, but preserve it.
      ch1 = getc
      buf << ch1.chr
    elsif ch == '"'
      if double
        return push_token(Token::String, buf)
      end
    elsif ch == "'"
      if !double
        return push_token(Token::String, buf)
      end
    end
#      push_token(Token::Char, buf)
  end

  warning("unterminated string literal", start_line, start_column)
  return nil
end

#shift_tokenObject

Used after a pop operation



222
223
224
# File 'lib/bijou/lexer.rb', line 222

def shift_token
  push_token @next_token, @next_text
end

#textObject



134
135
136
# File 'lib/bijou/lexer.rb', line 134

def text
  @text
end

#tokenObject



130
131
132
# File 'lib/bijou/lexer.rb', line 130

def token
  @token
end

#ungetc(ch) ⇒ Object

Used for backtracking, puts the most recently removed character back into the input stream.



155
156
157
# File 'lib/bijou/lexer.rb', line 155

def ungetc(ch)
  @input.ungetc(ch)
end

#warning(s, l = nil, c = nil) ⇒ Object



173
174
175
176
177
# File 'lib/bijou/lexer.rb', line 173

def warning(s, l=nil, c=nil)
  m = Bijou::Parse::Warning.new
  m << s
  @input.diagnostics.add_warning(diagnostic(m, l, c))
end

#warningsObject



192
193
194
# File 'lib/bijou/lexer.rb', line 192

def warnings()
  @input.diagnostics.warnings
end