Class: Datadog::CI::TestRetries::Component

Inherits:
Object
  • Object
show all
Defined in:
lib/datadog/ci/test_retries/component.rb

Overview

Encapsulates the logic to enable test retries, including:

  • retrying failed tests - improve success rate of CI pipelines

  • retrying new tests - detect flaky tests as early as possible to prevent them from being merged

Direct Known Subclasses

NullComponent

Constant Summary collapse

FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY = 
:__dd_current_retry_driver

Instance Method Summary collapse

Constructor Details

#initialize(retry_failed_tests_enabled:, retry_failed_tests_max_attempts:, retry_failed_tests_total_limit:, retry_new_tests_enabled:, retry_flaky_fixed_tests_enabled:, retry_flaky_fixed_tests_max_attempts:) ⇒ Component

Returns a new instance of Component.



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/datadog/ci/test_retries/component.rb', line 24

def initialize(
  retry_failed_tests_enabled:,
  retry_failed_tests_max_attempts:,
  retry_failed_tests_total_limit:,
  retry_new_tests_enabled:,
  retry_flaky_fixed_tests_enabled:,
  retry_flaky_fixed_tests_max_attempts:
)
  no_retries_strategy = Strategy::NoRetry.new

  retry_failed_strategy = Strategy::RetryFailed.new(
    enabled: retry_failed_tests_enabled,
    max_attempts: retry_failed_tests_max_attempts,
    total_limit: retry_failed_tests_total_limit
  )

  retry_flake_detection_strategy = Strategy::RetryFlakeDetection.new(
    enabled: retry_new_tests_enabled
  )

  retry_flaky_fixed_strategy = Strategy::RetryFlakyFixed.new(
    enabled: retry_flaky_fixed_tests_enabled,
    max_attempts: retry_flaky_fixed_tests_max_attempts
  )

  # order is important, we apply the first matching strategy
  @retry_strategies = [
    retry_flaky_fixed_strategy,
    retry_flake_detection_strategy,
    retry_failed_strategy,
    no_retries_strategy
  ]
  @mutex = Mutex.new
end

Instance Method Details

#build_driver(test_span) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
 
# File 'lib/datadog/ci/test_retries/component.rb', line 78

def build_driver(test_span)
  @mutex.synchronize do
    # find the first strategy that covers the test span and let it build the driver
    strategy = @retry_strategies.find { |strategy| strategy.covers?(test_span) }

    raise "No retry strategy found for test span: #{test_span.name}" if strategy.nil?

    strategy.build_driver(test_span)
  end
end

#configure(library_settings, test_session) ⇒ Object 



59
60
61
62
63
64
 
# File 'lib/datadog/ci/test_retries/component.rb', line 59

def configure(library_settings, test_session)
  # let all strategies configure themselves
  @retry_strategies.each do |strategy|
    strategy.configure(library_settings, test_session)
  end
end

#record_test_finished(test_span) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/datadog/ci/test_retries/component.rb', line 95

def record_test_finished(test_span)
  if current_retry_driver.nil?
    # we always run test at least once and after the first pass create a correct retry driver
    self.current_retry_driver = build_driver(test_span)
  else
    # after each retry we record the result, the driver will decide if we should retry again
    current_retry_driver&.record_retry(test_span)

    tag_last_retry(test_span) unless should_retry?
  end
end

#record_test_span_duration(tracer_span) ⇒ Object 



107
108
109
 
# File 'lib/datadog/ci/test_retries/component.rb', line 107

def record_test_span_duration(tracer_span)
  current_retry_driver&.record_duration(tracer_span.duration)
end

#record_test_started(test_span) ⇒ Object 



89
90
91
92
93
 
# File 'lib/datadog/ci/test_retries/component.rb', line 89

def record_test_started(test_span)
  # mark test as retry in the beginning
  # if this is a first execution, the current_retry_driver is nil and this is noop
  current_retry_driver&.mark_as_retry(test_span)
end

#reset_retries! ⇒ Object

this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method



112
113
114
 
# File 'lib/datadog/ci/test_retries/component.rb', line 112

def reset_retries!
  self.current_retry_driver = nil
end

#should_retry? ⇒ Boolean

Returns:

  • (Boolean) 


124
125
126
 
# File 'lib/datadog/ci/test_retries/component.rb', line 124

def should_retry?
  !!current_retry_driver&.should_retry?
end

#tag_last_retry(test_span) ⇒ Object 



116
117
118
119
120
121
122
 
# File 'lib/datadog/ci/test_retries/component.rb', line 116

def tag_last_retry(test_span)
  test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?

  # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
  # otherwise we send "false" to show that it didn't work
  test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, test_span&.all_executions_passed?.to_s) if test_span&.attempt_to_fix?
end

#with_retries(&block) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/datadog/ci/test_retries/component.rb', line 66

def with_retries(&block)
  reset_retries!

  loop do
    yield

    break unless should_retry?
  end
ensure
  reset_retries!
end