Class: Datadog::CI::TestRetries::Component

Inherits:
Object
  • Object
show all
Defined in:
lib/datadog/ci/test_retries/component.rb

Overview

Encapsulates the logic to enable test retries, including:

  • retrying failed tests - improve success rate of CI pipelines

  • retrying new tests - detect flaky tests as early as possible to prevent them from being merged

Direct Known Subclasses

NullComponent

Constant Summary collapse

FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY = 
:__dd_current_retry_driver

Instance Method Summary collapse

Constructor Details

#initialize(retry_failed_tests_enabled:, retry_failed_tests_max_attempts:, retry_failed_tests_total_limit:, retry_new_tests_enabled:, retry_flaky_fixed_tests_enabled:, retry_flaky_fixed_tests_max_attempts:) ⇒ Component

Returns a new instance of Component.



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/datadog/ci/test_retries/component.rb', line 24

def initialize(
  retry_failed_tests_enabled:,
  retry_failed_tests_max_attempts:,
  retry_failed_tests_total_limit:,
  retry_new_tests_enabled:,
  retry_flaky_fixed_tests_enabled:,
  retry_flaky_fixed_tests_max_attempts:
)
  no_retries_strategy = Strategy::NoRetry.new

  retry_failed_strategy = Strategy::RetryFailed.new(
    enabled: retry_failed_tests_enabled,
    max_attempts: retry_failed_tests_max_attempts,
    total_limit: retry_failed_tests_total_limit
  )

  retry_flake_detection_strategy = Strategy::RetryFlakeDetection.new(
    enabled: retry_new_tests_enabled
  )

  retry_flaky_fixed_strategy = Strategy::RetryFlakyFixed.new(
    enabled: retry_flaky_fixed_tests_enabled,
    max_attempts: retry_flaky_fixed_tests_max_attempts
  )

  # order is important, we apply the first matching strategy
  @retry_strategies = [
    retry_flaky_fixed_strategy,
    retry_flake_detection_strategy,
    retry_failed_strategy,
    no_retries_strategy
  ]
  @mutex = Mutex.new
end

Instance Method Details

#build_driver(test_span) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
 
# File 'lib/datadog/ci/test_retries/component.rb', line 78

def build_driver(test_span)
  @mutex.synchronize do
    # find the first strategy that covers the test span and let it build the driver
    strategy = @retry_strategies.find { |strategy| strategy.covers?(test_span) }

    raise "No retry strategy found for test span: #{test_span.name}" if strategy.nil?

    strategy.build_driver(test_span)
  end
end

#configure(library_settings, test_session) ⇒ Object 



59
60
61
62
63
64
 
# File 'lib/datadog/ci/test_retries/component.rb', line 59

def configure(library_settings, test_session)
  # let all strategies configure themselves
  @retry_strategies.each do |strategy|
    strategy.configure(library_settings, test_session)
  end
end

#record_test_finished(test_span) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/datadog/ci/test_retries/component.rb', line 95

def record_test_finished(test_span)
  if current_retry_driver.nil?
    # We always run test at least once and after the first pass create a correct retry driver
    self.current_retry_driver = build_driver(test_span)
  else
    # After each retry we let the driver to record the result.
    # Then the driver will decide if we should retry again.
    current_retry_driver&.record_retry(test_span)

    # We know that the test was already retried at least once so if we should not retry anymore, then this
    # is the last retry.
    tag_last_retry(test_span) unless should_retry?
  end

  # Some retry strategies such as Early Flake Detection change the number of retries based on
  # how long the test was.
  current_retry_driver&.record_duration(test_span.peek_duration)

  # We need to set the final status of the test (what will be reported to the test framework) on the last execution
  # no matter if test was retried or not
  #
  # If we should not retry at this point, it means that this execution is the last one (it might the only one as well).
  test_span.record_final_status unless should_retry?
end

#record_test_started(test_span) ⇒ Object 



89
90
91
92
93
 
# File 'lib/datadog/ci/test_retries/component.rb', line 89

def record_test_started(test_span)
  # mark test as retry in the beginning
  # if this is a first execution, the current_retry_driver is nil and this is noop
  current_retry_driver&.mark_as_retry(test_span)
end

#reset_retries! ⇒ Object

this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method



121
122
123
 
# File 'lib/datadog/ci/test_retries/component.rb', line 121

def reset_retries!
  self.current_retry_driver = nil
end

#should_retry? ⇒ Boolean

Returns:

  • (Boolean) 


133
134
135
 
# File 'lib/datadog/ci/test_retries/component.rb', line 133

def should_retry?
  !!current_retry_driver&.should_retry?
end

#tag_last_retry(test_span) ⇒ Object 



125
126
127
128
129
130
131
 
# File 'lib/datadog/ci/test_retries/component.rb', line 125

def tag_last_retry(test_span)
  test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?

  # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
  # otherwise we send "false" to show that it didn't work
  test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, test_span&.all_executions_passed?.to_s) if test_span&.attempt_to_fix?
end

#with_retries(&block) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/datadog/ci/test_retries/component.rb', line 66

def with_retries(&block)
  reset_retries!

  loop do
    yield

    break unless should_retry?
  end
ensure
  reset_retries!
end