Class: IRB::RubyLex

Inherits:
Object show all
Defined in:
lib/irb/ruby-lex.rb

Overview

:stopdoc:

Defined Under Namespace

Classes: TerminateLineInput

Constant Summary collapse

ASSIGNMENT_NODE_TYPES = 
[
  # Local, instance, global, class, constant, instance, and index assignment:
  #   "foo = bar",
  #   "@foo = bar",
  #   "$foo = bar",
  #   "@@foo = bar",
  #   "::Foo = bar",
  #   "a::Foo = bar",
  #   "Foo = bar"
  #   "foo.bar = 1"
  #   "foo[1] = bar"
  :assign,

  # Operation assignment:
  #   "foo += bar"
  #   "foo -= bar"
  #   "foo ||= bar"
  #   "foo &&= bar"
  :opassign,

  # Multiple assignment:
  #   "foo, bar = 1, 2
  :massign,
]
ERROR_TOKENS = 
[
  :on_parse_error,
  :compile_error,
  :on_assign_error,
  :on_alias_error,
  :on_class_name_error,
  :on_param_error
]
LTYPE_TOKENS = 
%i[
  on_heredoc_beg on_tstring_beg
  on_regexp_beg on_symbeg on_backtick
  on_symbols_beg on_qsymbols_beg
  on_words_beg on_qwords_beg
]
FREE_INDENT_TOKENS = 
%i[on_tstring_beg on_backtick on_regexp_beg on_symbeg]

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.compile_with_errors_suppressed(code, line_no: 1) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/irb/ruby-lex.rb', line 62

def compile_with_errors_suppressed(code, line_no: 1)
  begin
    result = yield code, line_no
  rescue ArgumentError
    # Ruby can issue an error for the code if there is an
    # incomplete magic comment for encoding in it. Force an
    # expression with a new line before the code in this
    # case to prevent magic comment handling.  To make sure
    # line numbers in the lexed code remain the same,
    # decrease the line number by one.
    code = ";\n#{code}"
    line_no -= 1
    result = yield code, line_no
  end
  result
end

.generate_local_variables_assign_code(local_variables) ⇒ Object 



79
80
81
 
# File 'lib/irb/ruby-lex.rb', line 79

def generate_local_variables_assign_code(local_variables)
  "#{local_variables.join('=')}=nil;" unless local_variables.empty?
end

.interpolate_ripper_ignored_tokens(code, tokens) ⇒ Object

Some part of the code is not included in Ripper’s token. Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr. With interpolated tokens, tokens.map(&:tok).join will be equal to code.



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/irb/ruby-lex.rb', line 86

def interpolate_ripper_ignored_tokens(code, tokens)
  line_positions = [0]
  code.lines.each do |line|
    line_positions << line_positions.last + line.bytesize
  end
  prev_byte_pos = 0
  interpolated = []
  prev_line = 1
  tokens.each do |t|
    line, col = t.pos
    byte_pos = line_positions[line - 1] + col
    if prev_byte_pos < byte_pos
      tok = code.byteslice(prev_byte_pos...byte_pos)
      pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
      interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
      prev_line += tok.count("\n")
    end
    interpolated << t
    prev_byte_pos = byte_pos + t.tok.bytesize
    prev_line += t.tok.count("\n")
  end
  if prev_byte_pos < code.bytesize
    tok = code.byteslice(prev_byte_pos..)
    pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
    interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
  end
  interpolated
end

.ripper_lex_without_warning(code, local_variables: []) ⇒ Object 



115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/irb/ruby-lex.rb', line 115

def ripper_lex_without_warning(code, local_variables: [])
  verbose, $VERBOSE = $VERBOSE, nil
  lvars_code = generate_local_variables_assign_code(local_variables)
  original_code = code
  if lvars_code
    code = "#{lvars_code}\n#{code}"
    line_no = 0
  else
    line_no = 1
  end

  compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no|
    lexer = Ripper::Lexer.new(inner_code, '-', line_no)
    tokens = []
    lexer.scan.each do |t|
      next if t.pos.first == 0
      prev_tk = tokens.last
      position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize
      if position_overlapped
        tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event)
      else
        tokens << t
      end
    end
    interpolate_ripper_ignored_tokens(original_code, tokens)
  end
ensure
  $VERBOSE = verbose
end

Instance Method Details

#assignment_expression?(code, local_variables:) ⇒ Boolean

Returns:

  • (Boolean) 


165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/irb/ruby-lex.rb', line 165

def assignment_expression?(code, local_variables:)
  # Try to parse the code and check if the last of possibly multiple
  # expressions is an assignment type.

  # If the expression is invalid, Ripper.sexp should return nil which will
  # result in false being returned. Any valid expression should return an
  # s-expression where the second element of the top level array is an
  # array of parsed expressions. The first element of each expression is the
  # expression's type.
  verbose, $VERBOSE = $VERBOSE, nil
  code = "#{RubyLex.generate_local_variables_assign_code(local_variables) || 'nil;'}\n#{code}"
  # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part.
  node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0)
  ASSIGNMENT_NODE_TYPES.include?(node_type)
ensure
  $VERBOSE = verbose
end

#calc_indent_level(opens) ⇒ Object 



283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'lib/irb/ruby-lex.rb', line 283

def calc_indent_level(opens)
  indent_level = 0
  opens.each_with_index do |t, index|
    case t.event
    when :on_heredoc_beg
      if opens[index + 1]&.event != :on_heredoc_beg
        if t.tok.match?(/^<<[~-]/)
          indent_level += 1
        else
          indent_level = 0
        end
      end
    when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick
      # No indent: "", //, :"", ``
      # Indent: %(), %r(), %i(), %x()
      indent_level += 1 if t.tok.start_with? '%'
    when :on_embdoc_beg
      indent_level = 0
    else
      indent_level += 1 unless t.tok == 'alias' || t.tok == 'undef'
    end
  end
  indent_level
end

#check_code_state(code, local_variables:) ⇒ Object 



146
147
148
149
150
 
# File 'lib/irb/ruby-lex.rb', line 146

def check_code_state(code, local_variables:)
  tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables)
  opens = NestingParser.open_tokens(tokens)
  [tokens, opens, code_terminated?(code, tokens, opens, local_variables: local_variables)]
end

#check_code_syntax(code, local_variables:) ⇒ Object 



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
 
# File 'lib/irb/ruby-lex.rb', line 207

def check_code_syntax(code, local_variables:)
  lvars_code = RubyLex.generate_local_variables_assign_code(local_variables)
  code = "#{lvars_code}\n#{code}"

  begin # check if parser error are available
    verbose, $VERBOSE = $VERBOSE, nil
    case RUBY_ENGINE
    when 'ruby'
      self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
        RubyVM::InstructionSequence.compile(inner_code, nil, nil, line_no)
      end
    when 'jruby'
      JRuby.compile_ir(code)
    else
      catch(:valid) do
        eval("BEGIN { throw :valid, true }\n#{code}")
        false
      end
    end
  rescue EncodingError
    # This is for a hash with invalid encoding symbol, {"\xAE": 1}
    :unrecoverable_error
  rescue SyntaxError => e
    case e.message
    when /unexpected keyword_end/
      # "syntax error, unexpected keyword_end"
      #
      #   example:
      #     if (
      #     end
      #
      #   example:
      #     end
      return :unrecoverable_error
    when /unexpected '\.'/
      # "syntax error, unexpected '.'"
      #
      #   example:
      #     .
      return :unrecoverable_error
    when /unexpected tREGEXP_BEG/
      # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
      #
      #   example:
      #     method / f /
      return :unrecoverable_error
    when /unterminated (?:string|regexp) meets end of file/
      # "unterminated regexp meets end of file"
      #
      #   example:
      #     /
      #
      # "unterminated string meets end of file"
      #
      #   example:
      #     '
      return :recoverable_error
    when /unexpected end-of-input/
      # "syntax error, unexpected end-of-input, expecting keyword_end"
      #
      #   example:
      #     if true
      #       hoge
      #       if false
      #         fuga
      #       end
      return :recoverable_error
    else
      return :other_error
    end
  ensure
    $VERBOSE = verbose
  end
  :valid
end

#check_termination_in_prev_line(code, local_variables:) ⇒ Object 



433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
 
# File 'lib/irb/ruby-lex.rb', line 433

def check_termination_in_prev_line(code, local_variables:)
  tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables)
  past_first_newline = false
  index = tokens.rindex do |t|
    # traverse first token before last line
    if past_first_newline
      if t.tok.include?("\n")
        true
      end
    elsif t.tok.include?("\n")
      past_first_newline = true
      false
    else
      false
    end
  end

  if index
    first_token = nil
    last_line_tokens = tokens[(index + 1)..(tokens.size - 1)]
    last_line_tokens.each do |t|
      unless [:on_sp, :on_ignored_sp, :on_comment].include?(t.event)
        first_token = t
        break
      end
    end

    if first_token && first_token.state != Ripper::EXPR_DOT
      tokens_without_last_line = tokens[0..index]
      code_without_last_line = tokens_without_last_line.map(&:tok).join
      opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line)
      if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line, local_variables: local_variables)
        return last_line_tokens.map(&:tok).join
      end
    end
  end
  false
end

#code_terminated?(code, tokens, opens, local_variables:) ⇒ Boolean

Returns:

  • (Boolean) 


152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/irb/ruby-lex.rb', line 152

def code_terminated?(code, tokens, opens, local_variables:)
  case check_code_syntax(code, local_variables: local_variables)
  when :unrecoverable_error
    true
  when :recoverable_error
    false
  when :other_error
    opens.empty? && !should_continue?(tokens)
  when :valid
    !should_continue?(tokens)
  end
end

#free_indent_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


310
311
312
 
# File 'lib/irb/ruby-lex.rb', line 310

def free_indent_token?(token)
  FREE_INDENT_TOKENS.include?(token&.event)
end

#indent_difference(lines, line_results, line_index) ⇒ Object

Calculates the difference of pasted code’s indent and indent calculated from tokens



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
 
# File 'lib/irb/ruby-lex.rb', line 315

def indent_difference(lines, line_results, line_index)
  loop do
    _tokens, prev_opens, _next_opens, min_depth = line_results[line_index]
    open_token = prev_opens.last
    if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token))
      # If the leading whitespace is an indent, return the difference
      indent_level = calc_indent_level(prev_opens.take(min_depth))
      calculated_indent = 2 * indent_level
      actual_indent = lines[line_index][/^ */].size
      return actual_indent - calculated_indent
    elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/)
      return 0
    end
    # If the leading whitespace is not an indent but part of a multiline token
    # Calculate base_indent of the multiline token's beginning line
    line_index = open_token.pos[0] - 1
  end
end

#ltype_from_open_tokens(opens) ⇒ Object 



403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
 
# File 'lib/irb/ruby-lex.rb', line 403

def ltype_from_open_tokens(opens)
  start_token = opens.reverse_each.find do |tok|
    LTYPE_TOKENS.include?(tok.event)
  end
  return nil unless start_token

  case start_token&.event
  when :on_tstring_beg
    case start_token&.tok
    when ?"      then ?"
    when /^%.$/  then ?"
    when /^%Q.$/ then ?"
    when ?'      then ?'
    when /^%q.$/ then ?'
    end
  when :on_regexp_beg   then ?/
  when :on_symbeg       then ?:
  when :on_backtick     then ?`
  when :on_qwords_beg   then ?]
  when :on_words_beg    then ?]
  when :on_qsymbols_beg then ?]
  when :on_symbols_beg  then ?]
  when :on_heredoc_beg
    start_token&.tok =~ /<<[-~]?(['"`])\w+\1/
    $1 || ?"
  else
    nil
  end
end

#process_indent_level(tokens, lines, line_index, is_newline) ⇒ Object 



334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
 
# File 'lib/irb/ruby-lex.rb', line 334

def process_indent_level(tokens, lines, line_index, is_newline)
  line_results = NestingParser.parse_by_line(tokens)
  result = line_results[line_index]
  if result
    _tokens, prev_opens, next_opens, min_depth = result
  else
    # When last line is empty
    prev_opens = next_opens = line_results.last[2]
    min_depth = next_opens.size
  end

  # To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation.
  # Shortest open tokens can be calculated by `opens.take(min_depth)`
  indent = 2 * calc_indent_level(prev_opens.take(min_depth))

  preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size

  prev_open_token = prev_opens.last
  next_open_token = next_opens.last

  # Calculates base indent for pasted code on the line where prev_open_token is located
  # irb(main):001:1*   if a # base_indent is 2, indent calculated from tokens is 0
  # irb(main):002:1*         if b # base_indent is 6, indent calculated from tokens is 2
  # irb(main):003:0>           c # base_indent is 6, indent calculated from tokens is 4
  if prev_open_token
    base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max
  else
    base_indent = 0
  end

  if free_indent_token?(prev_open_token)
    if is_newline && prev_open_token.pos[0] == line_index
      # First newline inside free-indent token
      base_indent + indent
    else
      # Accept any number of indent inside free-indent token
      preserve_indent
    end
  elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg
    if prev_open_token&.event == next_open_token&.event
      # Accept any number of indent inside embdoc content
      preserve_indent
    else
      # =begin or =end
      0
    end
  elsif prev_open_token&.event == :on_heredoc_beg
    tok = prev_open_token.tok
    if prev_opens.size <= next_opens.size
      if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token
        # First line in heredoc
        tok.match?(/^<<[-~]/) ? base_indent + indent : indent
      elsif tok.match?(/^<<~/)
        # Accept extra indent spaces inside `<<~` heredoc
        [base_indent + indent, preserve_indent].max
      else
        # Accept any number of indent inside other heredoc
        preserve_indent
      end
    else
      # Heredoc close
      prev_line_indent_level = calc_indent_level(prev_opens)
      tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0
    end
  else
    base_indent + indent
  end
end

#should_continue?(tokens) ⇒ Boolean

Returns:

  • (Boolean) 


183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/irb/ruby-lex.rb', line 183

def should_continue?(tokens)
  # Look at the last token and check if IRB need to continue reading next line.
  # Example code that should continue: `a\` `a +` `a.`
  # Trailing spaces, newline, comments are skipped
  return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n"

  tokens.reverse_each do |token|
    case token.event
    when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end
      # Skip
    when :on_regexp_end, :on_heredoc_end, :on_semicolon
      # State is EXPR_BEG but should not continue
      return false
    else
      # Endless range should not continue
      return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/)

      # EXPR_DOT and most of the EXPR_BEG should continue
      return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT)
    end
  end
  false
end