Class: Datadog::CI::TestRetries::Component

Inherits:
Object
  • Object
show all
Defined in:
lib/datadog/ci/test_retries/component.rb

Overview

Encapsulates the logic to enable test retries, including:

  • retrying failed tests - improve success rate of CI pipelines
  • retrying new tests - detect flaky tests as early as possible to prevent them from being merged

Direct Known Subclasses

NullComponent

Constant Summary collapse

FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY = 
:__dd_current_retry_driver

Instance Method Summary collapse

Constructor Details

#initialize(retry_failed_tests_enabled:, retry_failed_tests_max_attempts:, retry_failed_tests_total_limit:, retry_new_tests_enabled:, retry_flaky_fixed_tests_enabled:, retry_flaky_fixed_tests_max_attempts:, dynamic_atr_enabled:, dynamic_atr_buckets:) ⇒ Component

Returns a new instance of Component.



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/datadog/ci/test_retries/component.rb', line 25

def initialize(
  retry_failed_tests_enabled:,
  retry_failed_tests_max_attempts:,
  retry_failed_tests_total_limit:,
  retry_new_tests_enabled:,
  retry_flaky_fixed_tests_enabled:,
  retry_flaky_fixed_tests_max_attempts:,
  dynamic_atr_enabled:,
  dynamic_atr_buckets:
)
  no_retries_strategy = Strategy::NoRetry.new

  retry_failed_strategy = Strategy::RetryFailed.new(
    enabled: retry_failed_tests_enabled,
    max_attempts: retry_failed_tests_max_attempts,
    total_limit: retry_failed_tests_total_limit,
    dynamic_atr_enabled: dynamic_atr_enabled,
    dynamic_atr_buckets: dynamic_atr_buckets
  )

  retry_flake_detection_strategy = Strategy::RetryFlakeDetection.new(
    enabled: retry_new_tests_enabled
  )

  retry_flaky_fixed_strategy = Strategy::RetryFlakyFixed.new(
    enabled: retry_flaky_fixed_tests_enabled,
    max_attempts: retry_flaky_fixed_tests_max_attempts
  )

  # order is important, we apply the first matching strategy
  @retry_strategies = [
    retry_flaky_fixed_strategy,
    retry_flake_detection_strategy,
    retry_failed_strategy,
    no_retries_strategy
  ]
  @mutex = Mutex.new

  if dynamic_atr_enabled
    Utils::Telemetry.record_dynamic_atr_retries(has_custom_buckets: !dynamic_atr_buckets.nil?)
  end
end

Instance Method Details

#build_driver(test_span) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
 
# File 'lib/datadog/ci/test_retries/component.rb', line 87

def build_driver(test_span)
  @mutex.synchronize do
    # find the first strategy that covers the test span and let it build the driver
    strategy = @retry_strategies.find { |strategy| strategy.covers?(test_span) }

    raise "No retry strategy found for test span: #{test_span.name}" if strategy.nil?

    strategy.build_driver(test_span)
  end
end

#configure(library_settings, test_session) ⇒ Object 



68
69
70
71
72
73
 
# File 'lib/datadog/ci/test_retries/component.rb', line 68

def configure(library_settings, test_session)
  # let all strategies configure themselves
  @retry_strategies.each do |strategy|
    strategy.configure(library_settings, test_session)
  end
end

#record_test_finished(test_span) ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/datadog/ci/test_retries/component.rb', line 104

def record_test_finished(test_span)
  if current_retry_driver.nil?
    # We always run test at least once and after the first pass create a correct retry driver
    self.current_retry_driver = build_driver(test_span)

    # When the driver decides no retries are needed after the first execution
    # (e.g., attempt-to-fix test that failed immediately), we still need to tag
    # the outcome for strategies that track retry results.
    if !should_retry? && current_retry_driver&.tracks_retry_results?
      tag_last_retry(test_span)
    end
  else
    # After each retry we let the driver to record the result.
    # Then the driver will decide if we should retry again.
    current_retry_driver&.record_retry(test_span)

    # We know that the test was already retried at least once so if we should not retry anymore, then this
    # is the last retry.
    tag_last_retry(test_span) unless should_retry?
  end

  # Some retry strategies such as Early Flake Detection change the number of retries based on
  # how long the test was.
  current_retry_driver&.record_duration(test_span.peek_duration)

  # We need to set the final status of the test (what will be reported to the test framework) on the last execution
  # no matter if test was retried or not
  #
  # If we should not retry at this point, it means that this execution is the last one (it might the only one as well).
  test_span.record_final_status unless should_retry?
end

#record_test_started(test_span) ⇒ Object 



98
99
100
101
102
 
# File 'lib/datadog/ci/test_retries/component.rb', line 98

def record_test_started(test_span)
  # mark test as retry in the beginning
  # if this is a first execution, the current_retry_driver is nil and this is noop
  current_retry_driver&.mark_as_retry(test_span)
end

#reset_retries! ⇒ Object

this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method



137
138
139
 
# File 'lib/datadog/ci/test_retries/component.rb', line 137

def reset_retries!
  self.current_retry_driver = nil
end

#should_retry? ⇒ Boolean

Returns:

  • (Boolean) 


149
150
151
 
# File 'lib/datadog/ci/test_retries/component.rb', line 149

def should_retry?
  !!current_retry_driver&.should_retry?
end

#tag_last_retry(test_span) ⇒ Object 



141
142
143
144
145
146
147
 
# File 'lib/datadog/ci/test_retries/component.rb', line 141

def tag_last_retry(test_span)
  test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?

  # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
  # otherwise we send "false" to show that it didn't work
  test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, test_span&.all_executions_passed?.to_s) if test_span&.attempt_to_fix?
end

#with_retries(&block) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/datadog/ci/test_retries/component.rb', line 75

def with_retries(&block)
  reset_retries!

  loop do
    yield

    break unless should_retry?
  end
ensure
  reset_retries!
end