Module: WebHooks::AutoDisabling

Extended by:
ActiveSupport::Concern
Includes:
Gitlab::Loggable
Defined in:
app/models/concerns/web_hooks/auto_disabling.rb

Constant Summary collapse

ENABLED_HOOK_TYPES = 
%w[ProjectHook].freeze
TEMPORARILY_DISABLED_FAILURE_THRESHOLD = 
3
PERMANENTLY_DISABLED_FAILURE_THRESHOLD =

A webhook will be failing and being temporarily disabled for the max backoff of 1 day (MAX_BACKOFF) for at least 1 month before it becomes permanently disabled on its 40th failure. Exactly how quickly this happens depends on how frequently it triggers. gitlab.com/gitlab-org/gitlab/-/issues/503733#note_2217234805

39
INITIAL_BACKOFF = 
1.minute.freeze
MAX_BACKOFF = 
1.day.freeze
BACKOFF_GROWTH_FACTOR = 
2.0

Constants included from Gitlab::Loggable

Gitlab::Loggable::ANONYMOUS

Instance Method Summary collapse

Methods included from Gitlab::Loggable

#build_structured_payload

Instance Method Details

#alert_status ⇒ Object 



127
128
129
130
131
132
133
134
135
136
137
 
# File 'app/models/concerns/web_hooks/auto_disabling.rb', line 127

def alert_status
  return :executable unless auto_disabling_enabled?

  if temporarily_disabled?
    :temporarily_disabled
  elsif permanently_disabled?
    :disabled
  else
    :executable
  end
end

#backoff! ⇒ Object

Don’t actually back-off until a grace level of TEMPORARILY_DISABLED_FAILURE_THRESHOLD failures have been seen tracked in the recent_failures counter



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'app/models/concerns/web_hooks/auto_disabling.rb', line 111

def backoff!
  return unless auto_disabling_enabled? && executable?

  new_recent_failures = next_failure_count

  attrs = { recent_failures: new_recent_failures }
  attrs[:disabled_until] = next_backoff.from_now if new_recent_failures > TEMPORARILY_DISABLED_FAILURE_THRESHOLD

  assign_attributes(attrs)

  return unless changed?

  logger.info(hook_id: id, action: 'backoff', **attrs)
  save(validate: false)
end

#enable! ⇒ Object 



98
99
100
101
102
103
104
105
106
107
 
# File 'app/models/concerns/web_hooks/auto_disabling.rb', line 98

def enable!
  return unless auto_disabling_enabled?
  return if recent_failures == 0 && disabled_until.nil?

  attrs = { recent_failures: 0, disabled_until: nil }

  assign_attributes(attrs)
  logger.info(hook_id: id, action: 'enable', **attrs)
  save(validate: false)
end

#executable? ⇒ Boolean

Returns:

  • (Boolean) 


79
80
81
82
83
 
# File 'app/models/concerns/web_hooks/auto_disabling.rb', line 79

def executable?
  return true unless auto_disabling_enabled?

  !temporarily_disabled? && !permanently_disabled?
end

#permanently_disabled? ⇒ Boolean

Returns:

  • (Boolean) 


92
93
94
95
96
 
# File 'app/models/concerns/web_hooks/auto_disabling.rb', line 92

def permanently_disabled?
  return false unless auto_disabling_enabled?

  recent_failures > PERMANENTLY_DISABLED_FAILURE_THRESHOLD
end

#temporarily_disabled? ⇒ Boolean

Returns:

  • (Boolean) 


85
86
87
88
89
90
 
# File 'app/models/concerns/web_hooks/auto_disabling.rb', line 85

def temporarily_disabled?
  return false unless auto_disabling_enabled?

  disabled_until.present? && disabled_until >= Time.current &&
    recent_failures.between?(TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1, PERMANENTLY_DISABLED_FAILURE_THRESHOLD)
end