Module: Flaky

Defined in:
lib/flaky.rb,
lib/trace.rb,
lib/flaky/cmd.rb,
lib/flaky/run.rb,
lib/flaky/appium.rb,
lib/screen_recording.rb,
lib/flaky/run/one_test.rb,
lib/flaky/run/two_pass.rb,
lib/flaky/run/all_tests.rb,
lib/flaky/run/from_file.rb

Overview

Flaky.trace_specs trace: Dir.glob(File.join(__dir__, ‘**’, ‘*.rb’)) # verbose logging

Defined Under Namespace

Modules: Color Classes: Appium, Cmd, LogArtifact, Run

Constant Summary collapse

VERSION = 
'0.1.3'
DATE = 
'2015-04-28'

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.no_video ⇒ Object

Returns the value of attribute no_video.



19
20
21
 
# File 'lib/flaky.rb', line 19

def no_video
  @no_video
end

Class Method Details

.capture_ios_app_log(app_name) ⇒ Object

app_name for example MyApp.app



12
13
14
 
# File 'lib/screen_recording.rb', line 12

def capture_ios_app_log app_name
  # nop -- this feature has moved into the appium server
end

.run_all_tests(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/flaky/run/all_tests.rb', line 3

def self.run_all_tests opts={}
  raise 'Must pass :count and :os' unless opts && opts[:count] && opts[:os]

  count = opts[:count].to_i
  os = opts[:os]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  flaky = Flaky::Run.new
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  current_dir = Dir.pwd
  rakefile = File.expand_path(File.join(current_dir, 'Rakefile'))
  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists? rakefile
  flaky_txt = File.expand_path(File.join(current_dir, 'flaky.txt'))
  parsed = TOML.load File.read flaky_txt
  puts "flaky.txt: #{parsed}"
  android_dir = parsed['android']
  ios_dir = parsed['ios']
  glob = parsed.fetch 'glob', '**/*.rb'

  active_dir = is_android ? android_dir : ios_dir
  final_path = File.expand_path File.join current_dir, active_dir, glob
  puts "Globbing: #{final_path}"

  Dir.glob(final_path) do |test_file|
    raise "#{test_file} does not exist." unless File.exist?(test_file)
    test_file = File.expand_path test_file

    test_name = test_file.sub(File.expand_path(File.join(current_dir, active_dir)), '')
    # remove leading /
    test_name.sub!(test_name.match(/^\//).to_s, '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count.times do
      File.open('/tmp/flaky/current.txt', 'a') { |f| f.puts "Running: #{test_name} on #{os}" }
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{test_file}',#{Flaky.no_video}]"
      passed = flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report
end

.run_from_file(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/flaky/run/from_file.rb', line 3

def self.run_from_file opts={}
  raise 'Must pass :count, :os, and :file' unless opts && opts[:count] && opts[:os] && opts[:file]

  count = opts[:count].to_i
  os = opts[:os]
  file = opts[:file]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)
  raise ':file must be a string' unless file.kind_of?(String)

  raise "#{file} doesn't exist" unless File.exists? file
  tests = File.readlines(file).map { |line| File.basename(line.chomp, '.*') }
  resolved_paths = []
  # Convert file names into full paths
  current_dir = Dir.pwd
  Dir.glob(File.join current_dir, 'appium', os, 'specs', '**/*.rb') do |test_file|
    if tests.include? File.basename(test_file, '.*')
      resolved_paths << File.expand_path(test_file)
    end
  end

  if tests.length != resolved_paths.length
    missing_tests = []
    tests.each do |test|
      missing_tests << test unless File.exists? test
    end
    raise "Missing tests #{missing_tests}"
  end

  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists?(File.join(current_dir, 'Rakefile'))

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  flaky = Flaky::Run.new
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  resolved_paths.each do |test_file|
    file = test_file
    name = File.basename file, '.*'

    raise "#{test_file} does not exist." if file.empty?

    test_name = file.sub(current_dir + '/appium/', '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count.times do
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{name}',#{Flaky.no_video}]"
      passed = flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report
end

.run_one_test(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/flaky/run/one_test.rb', line 3

def self.run_one_test opts={}
  raise 'Must pass :count and :name' unless opts && opts[:count] && opts[:os] && opts[:name]

  count = opts[:count].to_i
  os = opts[:os]
  name = opts[:name]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)
  raise ':name must be a string' unless name.kind_of?(String)

  # ensure file name does not contain an extension
  # don't expand the path because it's joined and expanded in final_path.
  name = File.join(File.dirname(name), File.basename(name, '.*'))

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  flaky = Flaky::Run.new
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  current_dir = Dir.pwd

  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists?(File.join(current_dir, 'Rakefile'))
  flaky_txt = File.expand_path(File.join(current_dir, 'flaky.txt'))
  parsed = TOML.load File.read flaky_txt
  puts "flaky.txt: #{parsed}"
  android_dir = parsed['android']
  ios_dir = parsed['ios']
  active_dir = is_android ? android_dir : ios_dir
  final_path = File.expand_path File.join current_dir, active_dir, name + '.rb'
  test_file = ''
  Dir.glob(final_path) do |file|
    test_file = file
  end

  raise "#{test_file} does not exist." unless File.exists?(test_file)

  test_name = test_file.sub(File.expand_path(File.join(current_dir, active_dir)), '')
  # remove leading /
  test_name.sub!(test_name.match(/^\//).to_s, '')
  test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

  count.times do
    appium.start unless running_on_sauce
    run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{test_file}',#{Flaky.no_video}]"
    flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
  end

  appium.stop unless running_on_sauce
  flaky.report
end

.screen_recording_binary ⇒ Object 



16
17
18
 
# File 'lib/screen_recording.rb', line 16

def screen_recording_binary
  @screen_recording_binary ||= File.expand_path('../screen-recording', __FILE__)
end

.screen_recording_start(opts = {}) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/screen_recording.rb', line 20

def screen_recording_start opts={}
  return if Flaky.no_video

  os = opts[:os]
  path = opts[:path]
  raise ':os is required' unless os
  raise ':path is required' unless path

  raise 'Invalid os. Must be ios or android' unless %w[ios android].include? os
  raise 'Invalid path. Must end with .mov' unless File.extname(path) == '.mov'
  raise 'Invalid path. Must not be a dir' if File.exists?(path) && File.directory?(path)

  # ensure we have exactly one screen-recording process
  # wait for killall to complete
  Process::waitpid(spawn('killall', '-9', 'screen-recording', :in => '/dev/null', :out => '/dev/null', :err => '/dev/null'))

  File.delete(path) if File.exists? path

  pid = spawn(screen_recording_binary, os, path,
              :in => '/dev/null', :out => '/dev/null', :err => '/dev/null')
  pid
end

.screen_recording_stop(pid) ⇒ Object 



43
44
45
46
47
48
49
 
# File 'lib/screen_recording.rb', line 43

def screen_recording_stop pid
  Process.kill(:SIGINT, pid)
  # Must wait 5 seconds for the video to end.
  # If we don't wait, the movie will be corrupt.
  # See: https://github.com/bootstraponline/screen_recording/blob/master/screen-recording/main.m#L137
  sleep 5
end

.trace_specs(spec_opts) ⇒ Object

Trace file source to :io (default $stdout)

spec_opts = {}

Parameters:

  • :trace (Array<String>) —

    the files to trace

  • :io (IO) —

    io to print to



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/trace.rb', line 10

def self.trace_specs spec_opts
  targets   = []
  files     = {}
  last_file = ''
  last_line = -1

  files_to_trace = spec_opts.fetch(:trace, []);
  io    = spec_opts.fetch(:io, $stdout)
  color = spec_opts.fetch(:color, "\e[32m") # ANSI.green default
  # target only existing readable files
  files_to_trace.each do |f|
    if File.exists?(f) && File.readable?(f)
      targets.push File.expand_path f
      targets.push File.basename f # sometimes the file is relative
    end
  end
  return if targets.empty?

  set_trace_func(lambda do |event, file, line, id, binding, classname|
    return unless targets.include?(file)

    # never repeat a line
    return if file == last_file && line == last_line

    file_sym        = file.intern
    files[file_sym] = IO.readlines(file) if files[file_sym].nil?
    lines           = files[file_sym]

    # arrays are 0 indexed and line numbers start at one.
    io.print color if color # ANSI code
    io.puts lines[line - 1]
    io.print "\e[0m" if color # ANSI.clear

    last_file = file
    last_line = line

  end)
end

.two_pass(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/flaky/run/two_pass.rb', line 3

def self.two_pass opts={}
  raise 'Must pass :count and :os' unless opts && opts[:count] && opts[:os]

  count = opts[:count].to_i
  os = opts[:os]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)

  count1 = 1
  count2 = count

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  FileUtils.rm_rf '/tmp/flaky'
  result_dir_postfix = '1' # /tmp/flaky/1
  flaky = Flaky::Run.new(result_dir_postfix)
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  current_dir = Dir.pwd
  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists?(File.join(current_dir, 'Rakefile'))

  # run all tests once
  Dir.glob(File.join current_dir, 'appium', os, 'specs', '**/*.rb') do |test_file|
    file = test_file
    name = File.basename file, '.*'

    raise "#{test_file} does not exist." if file.empty?

    test_name = file.sub(current_dir + '/appium/', '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count1.times do
      File.open('/tmp/flaky/current.txt', 'a') { |f| f.puts "Running: #{test_name} on #{os}" }
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{name}',#{Flaky.no_video}]"
      passed = flaky.execute(run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce)
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report

  # ---

  # now run only the failures count2 times
  fails = File.read(File.join('/tmp/flaky/', result_dir_postfix, 'fail.txt'))

  result_dir_postfix = '2' # /tmp/flaky/1
  flaky = Flaky::Run.new(result_dir_postfix)
  appium = Appium.new(android: is_android) unless running_on_sauce

  fails.split("\n").each do |test_file|
    file = test_file
    name = File.basename file, '.*'

    raise "#{test_file} does not exist." if file.empty?

    test_name = file.sub(current_dir + '/appium/', '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count2.times do
      File.open('/tmp/flaky/current.txt', 'a') { |f| f.puts "Running: #{test_name} on #{os}" }
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{name}',#{Flaky.no_video}]"
      passed = flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report
end