Class: Minitest::Bisect

Inherits:
Object
  • Object
show all
Defined in:
lib/minitest/bisect.rb

Overview

Minitest::Bisect helps you isolate and debug random test failures.

Defined Under Namespace

Classes: PathExpander

Constant Summary collapse

VERSION =

:nodoc:

"1.7.0"
SHH =

:nodoc:

case # :nodoc:
when mtbv == 1 then " > /dev/null"
when mtbv >= 2 then nil
else " > /dev/null 2>&1"
end
RUBY =

Borrowed from rake

ENV['RUBY'] ||
    File.join(RbConfig::CONFIG['bindir'],
              RbConfig::CONFIG['ruby_install_name'] +
RbConfig::CONFIG['EXEEXT']).sub(/.*\s.*/m, '"\&"')

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Bisect

Instantiate a new Bisect.



93
94
95
96
 
# File 'lib/minitest/bisect.rb', line 93

def initialize
  self.culprits = []
  self.failures = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = [] } }
end

Instance Attribute Details

#culprits ⇒ Object

An array of tests seen so far. NOT cleared by #reset.



75
76
77
 
# File 'lib/minitest/bisect.rb', line 75

def culprits
  @culprits
end

#failures ⇒ Object

Failures seen in this run. Shape:

{"file.rb"=>{"Class"=>["test_method1", "test_method2"] ...} ...}



70
71
72
 
# File 'lib/minitest/bisect.rb', line 70

def failures
  @failures
end

#seen_bad ⇒ Object

:nodoc:



77
78
79
 
# File 'lib/minitest/bisect.rb', line 77

def seen_bad
  @seen_bad
end

#tainted ⇒ Object Also known as: tainted?

True if this run has seen a failure.



62
63
64
 
# File 'lib/minitest/bisect.rb', line 62

def tainted
  @tainted
end

Class Method Details

.run(files) ⇒ Object

Top-level runner. Instantiate and call run, handling exceptions.



82
83
84
85
86
87
88
 
# File 'lib/minitest/bisect.rb', line 82

def self.run files
  new.run files
rescue => e
  warn e.message
  warn "Try running with MTB_VERBOSE=2 to verify."
  exit 1
end

Instance Method Details

#bisect_methods(files, rb_flags, mt_flags) ⇒ Object

Normal: find “what is the minimal combination of tests to run to

make X fail?"

Run with: minitest_bisect … –seed=N

  1. Verify the failure running normally with the seed.

    1. If no failure, punt.

    2. If no passing tests before failure, punt. (No culprits == no debug)

  2. Verify the failure doesn’t fail in isolation.

    1. If it still fails by itself, warn that it might not be an ordering issue.

  3. Cull all tests after the failure, they’re not involved.

  4. Bisect the culprits + bad until you find a minimal combo that fails.

  5. Display minimal combo by running one last time.

Inverted: find “what is the minimal combination of tests to run to

make this test pass?"

Run with: minitest_bisect … –seed=N -n=“/failing_test_name_regexp/”

  1. Verify the failure by running normally w/ the seed and -n=/…/

    1. If no failure, punt.

  2. Verify the passing case by running everything.

    1. If failure, punt. This is not a false positive.

  3. Cull all tests after the bad test from #1, they’re not involved.

  4. Bisect the culprits + bad until you find a minimal combo that passes.

  5. Display minimal combo by running one last time.



163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/minitest/bisect.rb', line 163

def bisect_methods files, rb_flags, mt_flags
  bad_names, mt_flags = mt_flags.partition { |s| s =~ /^(?:-n|--name)/ }
  normal   = bad_names.empty?
  inverted = !normal

  if inverted then
    time_it "reproducing w/ scoped failure (inverted run!)...", build_methods_cmd(build_files_cmd(files, rb_flags, mt_flags + bad_names))
    raise "No failures. Probably not a false positive. Aborting." if failures.empty?
    bad = map_failures
  end

  cmd = build_files_cmd(files, rb_flags, mt_flags)

  msg = normal ? "reproducing..." : "reproducing false positive..."
  time_it msg, build_methods_cmd(cmd)

  if normal then
    raise "Reproduction run passed? Aborting." unless tainted?
    raise "Verification failed. No culprits? Aborting." if culprits.empty? && seen_bad
  else
    raise "Reproduction failed? Not false positive. Aborting." if tainted?
    raise "Verification failed. No culprits? Aborting." if culprits.empty? || seen_bad
  end

  if normal then
    bad = map_failures

    time_it "verifying...", build_methods_cmd(cmd, [], bad)

    new_bad = map_failures

    if bad == new_bad then
      warn "Tests fail by themselves. This may not be an ordering issue."
    end
  end

  idx = culprits.index bad.first
  self.culprits = culprits.take idx+1 if idx # cull tests after bad

  # culprits populated by initial reproduction via minitest/server
  found, count = culprits.find_minimal_combination_and_count do |test|
    prompt = "# of culprit methods: #{test.size}"

    time_it prompt, build_methods_cmd(cmd, test, bad)

    normal == tainted? # either normal and failed, or inverse and passed
  end

  puts
  puts "Minimal methods found in #{count} steps:"
  puts
  puts "Culprit methods: %p" % [found + bad]
  puts
  cmd = build_methods_cmd cmd, found, bad
  puts cmd.sub(/--server \d+/, "")
  puts
  cmd
end

#build_files_cmd(culprits, rb, mt) ⇒ Object

:nodoc:



237
238
239
240
241
 
# File 'lib/minitest/bisect.rb', line 237

def build_files_cmd culprits, rb, mt # :nodoc:
  tests = culprits.flatten.compact.map { |f| %(require "./#{f}") }.join " ; "

  %(#{RUBY} #{rb.shelljoin} -e '#{tests}' -- #{mt.map(&:to_s).shelljoin})
end

#build_methods_cmd(cmd, culprits = [], bad = nil) ⇒ Object

:nodoc:



243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/minitest/bisect.rb', line 243

def build_methods_cmd cmd, culprits = [], bad = nil # :nodoc:
  reset

  if bad then
    re = build_re culprits + bad

    cmd += " -n \"#{re}\"" if bad
  end

  if ENV["MTB_VERBOSE"].to_i >= 1 then
    puts
    puts cmd
    puts
  end

  cmd
end

#build_re(bad) ⇒ Object

:nodoc:



261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/minitest/bisect.rb', line 261

def build_re bad # :nodoc:
  re = []

  # bad by class, you perv
  bbc = bad.map { |s| s.split(/#/, 2) }.group_by(&:first)

  bbc.each do |klass, methods|
    methods = methods.map(&:last).flatten.uniq.map { |method|
      re_escape method
    }

    methods = methods.join "|"
    re << /#{re_escape klass}#(?:#{methods})/.to_s[7..-2] # (?-mix:...)
  end

  re = re.join("|").to_s.gsub(/-mix/, "")

  "/^(?:#{re})$/"
end

#map_failures ⇒ Object

:nodoc:



229
230
231
232
233
234
235
 
# File 'lib/minitest/bisect.rb', line 229

def map_failures # :nodoc:
  # from: {"file.rb"=>{"Class"=>["test_method1", "test_method2"]}}
  #   to: ["Class#test_method1", "Class#test_method2"]
  failures.values.map { |h|
    h.map { |k,vs| vs.map { |v| "#{k}##{v}" } }
  }.flatten.sort
end

#minitest_result(file, klass, method, fails, assertions, time) ⇒ Object

:nodoc:



292
293
294
295
296
297
298
299
300
301
302
303
304
305
 
# File 'lib/minitest/bisect.rb', line 292

def minitest_result file, klass, method, fails, assertions, time # :nodoc:
  fails.reject! { |fail| Minitest::Skip === fail }

  if fails.empty? then
    culprits << "#{klass}##{method}" unless seen_bad # UGH
  else
    self.seen_bad = true
  end

  return if fails.empty?

  self.tainted = true
  self.failures[file][klass] << method
end

#minitest_start ⇒ Object

Server Methods:



288
289
290
 
# File 'lib/minitest/bisect.rb', line 288

def minitest_start # :nodoc:
  self.failures.clear
end

#re_escape(str) ⇒ Object

:nodoc:



281
282
283
 
# File 'lib/minitest/bisect.rb', line 281

def re_escape str # :nodoc:
  str.gsub(/([`'"!?&\[\]\(\)\{\}\|\+])/, '\\\\\1')
end

#reset ⇒ Object

Reset per-bisect-run variables.



101
102
103
104
105
106
 
# File 'lib/minitest/bisect.rb', line 101

def reset
  self.seen_bad = false
  self.tainted  = false
  failures.clear
  # not clearing culprits on purpose
end

#run(args) ⇒ Object

Instance-level runner. Handles Minitest::Server, argument processing, and invoking bisect_methods.



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/minitest/bisect.rb', line 112

def run args
  Minitest::Server.run self

  cmd = nil

  mt_flags = args.dup
  expander = Minitest::Bisect::PathExpander.new mt_flags

  files = expander.process
  rb_flags = expander.rb_flags
  mt_flags += ["--server", $$.to_s]

  cmd = bisect_methods files, rb_flags, mt_flags

  puts "Final reproduction:"
  puts

  system cmd.sub(/--server \d+/, "")
ensure
  Minitest::Server.stop
end

#time_it(prompt, cmd) ⇒ Object

:nodoc:



222
223
224
225
226
227
 
# File 'lib/minitest/bisect.rb', line 222

def time_it prompt, cmd # :nodoc:
  print prompt
  t0 = Time.now
  system "#{cmd} #{SHH}"
  puts " in %.2f sec" % (Time.now - t0)
end