Module: IRB::NestingParser

Defined in:
lib/irb/nesting_parser.rb

Constant Summary collapse

IGNORE_TOKENS = 
%i[on_sp on_ignored_nl on_comment on_embdoc_beg on_embdoc on_embdoc_end]

Class Method Summary collapse

Class Method Details

.open_tokens(tokens) ⇒ Object 



192
193
194
195
 
# File 'lib/irb/nesting_parser.rb', line 192

def self.open_tokens(tokens)
  # scan_opens without block will return a list of open tokens at last token position
  scan_opens(tokens)
end

.parse_by_line(tokens) ⇒ Object

Calculates token information [line_tokens, prev_opens, next_opens, min_depth] for each line. Example code

["hello
world"+(

First line

line_tokens: [[lbracket, '['], [tstring_beg, '"'], [tstring_content("hello\nworld"), "hello\n"]]
prev_opens:  []
next_tokens: [lbracket, tstring_beg]
min_depth:   0 (minimum at beginning of line)

Second line

line_tokens: [[tstring_content("hello\nworld"), "world"], [tstring_end, '"'], [op, '+'], [lparen, '(']]
prev_opens:  [lbracket, tstring_beg]
next_tokens: [lbracket, lparen]
min_depth:   1 (minimum just after tstring_end)



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/irb/nesting_parser.rb', line 211

def self.parse_by_line(tokens)
  line_tokens = []
  prev_opens = []
  min_depth = 0
  output = []
  last_opens = scan_opens(tokens) do |t, opens|
    depth = t == opens.last&.first ? opens.size - 1 : opens.size
    min_depth = depth if depth < min_depth
    if t.tok.include?("\n")
      t.tok.each_line do |line|
        line_tokens << [t, line]
        next if line[-1] != "\n"
        next_opens = opens.map(&:first)
        output << [line_tokens, prev_opens, next_opens, min_depth]
        prev_opens = next_opens
        min_depth = prev_opens.size
        line_tokens = []
      end
    else
      line_tokens << [t, t.tok]
    end
  end
  output << [line_tokens, prev_opens, last_opens, min_depth] if line_tokens.any?
  output
end

.scan_opens(tokens) ⇒ Object

Scan each token and call the given block with array of token and other information for parsing



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/irb/nesting_parser.rb', line 7

def self.scan_opens(tokens)
  opens = []
  pending_heredocs = []
  first_token_on_line = true
  tokens.each do |t|
    skip = false
    last_tok, state, args = opens.last
    case state
    when :in_alias_undef
      skip = t.event == :on_kw
    when :in_unquoted_symbol
      unless IGNORE_TOKENS.include?(t.event)
        opens.pop
        skip = true
      end
    when :in_lambda_head
      opens.pop if t.event == :on_tlambeg || (t.event == :on_kw && t.tok == 'do')
    when :in_method_head
      unless IGNORE_TOKENS.include?(t.event)
        next_args = []
        body = nil
        if args.include?(:receiver)
          case t.event
          when :on_lparen, :on_ivar, :on_gvar, :on_cvar
            # def (receiver). | def @ivar. | def $gvar. | def @@cvar.
            next_args << :dot
          when :on_kw
            case t.tok
            when 'self', 'true', 'false', 'nil'
              # def self(arg) | def self.
              next_args.push(:arg, :dot)
            else
              # def if(arg)
              skip = true
              next_args << :arg
            end
          when :on_op, :on_backtick
            # def +(arg)
            skip = true
            next_args << :arg
          when :on_ident, :on_const
            # def a(arg) | def a.
            next_args.push(:arg, :dot)
          end
        end
        if args.include?(:dot)
          # def receiver.name
          next_args << :name if t.event == :on_period || (t.event == :on_op && t.tok == '::')
        end
        if args.include?(:name)
          if %i[on_ident on_const on_op on_kw on_backtick].include?(t.event)
            # def name(arg) | def receiver.name(arg)
            next_args << :arg
            skip = true
          end
        end
        if args.include?(:arg)
          case t.event
          when :on_nl, :on_semicolon
            # def receiver.f;
            body = :normal
          when :on_lparen
            # def receiver.f()
            next_args << :eq
          else
            if t.event == :on_op && t.tok == '='
              # def receiver.f =
              body = :oneliner
            else
              # def receiver.f arg
              next_args << :arg_without_paren
            end
          end
        end
        if args.include?(:eq)
          if t.event == :on_op && t.tok == '='
            body = :oneliner
          else
            body = :normal
          end
        end
        if args.include?(:arg_without_paren)
          if %i[on_semicolon on_nl].include?(t.event)
            # def f a;
            body = :normal
          else
            # def f a, b
            next_args << :arg_without_paren
          end
        end
        if body == :oneliner
          opens.pop
        elsif body
          opens[-1] = [last_tok, nil]
        else
          opens[-1] = [last_tok, :in_method_head, next_args]
        end
      end
    when :in_for_while_until_condition
      if t.event == :on_semicolon || t.event == :on_nl || (t.event == :on_kw && t.tok == 'do')
        skip = true if t.event == :on_kw && t.tok == 'do'
        opens[-1] = [last_tok, nil]
      end
    end

    unless skip
      case t.event
      when :on_kw
        case t.tok
        when 'begin', 'class', 'module', 'do', 'case'
          opens << [t, nil]
        when 'end'
          opens.pop
        when 'def'
          opens << [t, :in_method_head, [:receiver, :name]]
        when 'if', 'unless'
          unless t.state.allbits?(Ripper::EXPR_LABEL)
            opens << [t, nil]
          end
        when 'while', 'until'
          unless t.state.allbits?(Ripper::EXPR_LABEL)
            opens << [t, :in_for_while_until_condition]
          end
        when 'ensure', 'rescue'
          unless t.state.allbits?(Ripper::EXPR_LABEL)
            opens.pop
            opens << [t, nil]
          end
        when 'alias'
          opens << [t, :in_alias_undef, 2]
        when 'undef'
          opens << [t, :in_alias_undef, 1]
        when 'elsif', 'else', 'when'
          opens.pop
          opens << [t, nil]
        when 'for'
          opens << [t, :in_for_while_until_condition]
        when 'in'
          if last_tok&.event == :on_kw && %w[case in].include?(last_tok.tok) && first_token_on_line
            opens.pop
            opens << [t, nil]
          end
        end
      when :on_tlambda
        opens << [t, :in_lambda_head]
      when :on_lparen, :on_lbracket, :on_lbrace, :on_tlambeg, :on_embexpr_beg, :on_embdoc_beg
        opens << [t, nil]
      when :on_rparen, :on_rbracket, :on_rbrace, :on_embexpr_end, :on_embdoc_end
        opens.pop
      when :on_heredoc_beg
        pending_heredocs << t
      when :on_heredoc_end
        opens.pop
      when :on_backtick
        opens << [t, nil] if t.state.allbits?(Ripper::EXPR_BEG)
      when :on_tstring_beg, :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_regexp_beg
        opens << [t, nil]
      when :on_tstring_end, :on_regexp_end, :on_label_end
        opens.pop
      when :on_symbeg
        if t.tok == ':'
          opens << [t, :in_unquoted_symbol]
        else
          opens << [t, nil]
        end
      end
    end
    if t.event == :on_nl || t.event == :on_semicolon
      first_token_on_line = true
    elsif t.event != :on_sp
      first_token_on_line = false
    end
    if pending_heredocs.any? && t.tok.include?("\n")
      pending_heredocs.reverse_each { |t| opens << [t, nil] }
      pending_heredocs = []
    end
    if opens.last && opens.last[1] == :in_alias_undef && !IGNORE_TOKENS.include?(t.event) && t.event != :on_heredoc_end
      tok, state, arg = opens.pop
      opens << [tok, state, arg - 1] if arg >= 1
    end
    yield t, opens if block_given?
  end
  opens.map(&:first) + pending_heredocs.reverse
end