Class: KubernetesDeploy::KubernetesResource

Inherits:
Object
  • Object
show all
Defined in:
lib/kubernetes-deploy/kubernetes_resource.rb

Defined Under Namespace

Classes: Event

Constant Summary collapse

GLOBAL = 
false
TIMEOUT = 
5.minutes
LOG_LINE_COUNT = 
250
DISABLE_FETCHING_LOG_INFO = 
'DISABLE_FETCHING_LOG_INFO'
DISABLE_FETCHING_EVENT_INFO = 
'DISABLE_FETCHING_EVENT_INFO'
DISABLED_LOG_INFO_MESSAGE = 
"collection is disabled by the #{DISABLE_FETCHING_LOG_INFO} env var."
DISABLED_EVENT_INFO_MESSAGE = 
"collection is disabled by the #{DISABLE_FETCHING_EVENT_INFO} env var."
DEBUG_RESOURCE_NOT_FOUND_MESSAGE = 
"None found. Please check your usual logging service (e.g. Splunk)."
UNUSUAL_FAILURE_MESSAGE = 
<<~MSG
It is very unusual for this resource type to fail to deploy. Please try the deploy again.
If that new deploy also fails, contact your cluster administrator.
MSG
STANDARD_TIMEOUT_MESSAGE = 
<<~MSG
Kubernetes will continue to attempt to deploy this resource in the cluster, but at this point it is considered unlikely that it will succeed.
If you have reason to believe it will succeed, retry the deploy to continue to monitor the rollout.
MSG
TIMEOUT_OVERRIDE_ANNOTATION = 
"kubernetes-deploy.shopify.io/timeout-override"
LAST_APPLIED_ANNOTATION = 
"kubectl.kubernetes.io/last-applied-configuration"
KUBECTL_OUTPUT_IS_SENSITIVE = 
false

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(namespace:, context:, definition:, logger:, statsd_tags: []) ⇒ KubernetesResource

Returns a new instance of KubernetesResource.



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 87

def initialize(namespace:, context:, definition:, logger:, statsd_tags: [])
  # subclasses must also set these if they define their own initializer
  @name = definition.dig("metadata", "name")
  unless @name.present?
    logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
    raise FatalDeploymentError, "Template is missing required field metadata.name"
  end

  @optional_statsd_tags = statsd_tags
  @namespace = namespace
  @context = context
  @logger = logger
  @definition = definition
  @statsd_report_done = false
  @disappeared = false
  @validation_errors = []
  @instance_data = {}
end

Instance Attribute Details

#context ⇒ Object (readonly)

Returns the value of attribute context.



10
11
12
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 10

def context
  @context
end

#deploy_started_at=(value) ⇒ Object (writeonly)

Sets the attribute deploy_started_at

Parameters:

  • value —

    the value to set the attribute deploy_started_at to.



11
12
13
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 11

def deploy_started_at=(value)
  @deploy_started_at = value
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



10
11
12
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 10

def name
  @name
end

#namespace ⇒ Object (readonly)

Returns the value of attribute namespace.



10
11
12
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 10

def namespace
  @namespace
end

#type ⇒ Object 



202
203
204
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 202

def type
  @type || self.class.kind
end

Class Method Details

.build(namespace:, context:, definition:, logger:, statsd_tags:, crd: nil) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 36

def build(namespace:, context:, definition:, logger:, statsd_tags:, crd: nil)
  opts = { namespace: namespace, context: context, definition: definition, logger: logger,
           statsd_tags: statsd_tags }
  if definition["kind"].blank?
    raise InvalidTemplateError.new("Template missing 'Kind'", content: definition.to_yaml)
  end
  if (klass = class_for_kind(definition["kind"]))
    return klass.new(**opts)
  end
  if crd
    CustomResource.new(crd: crd, **opts)
  else
    inst = new(**opts)
    inst.type = definition["kind"]
    inst
  end
end

.class_for_kind(kind) ⇒ Object 



54
55
56
57
58
59
60
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 54

def class_for_kind(kind)
  if KubernetesDeploy.const_defined?(kind)
    KubernetesDeploy.const_get(kind) # rubocop:disable Sorbet/ConstantsFromStrings
  end
rescue NameError
  nil
end

.kind ⇒ Object 



66
67
68
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 66

def kind
  name.demodulize
end

.timeout ⇒ Object 



62
63
64
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 62

def timeout
  self::TIMEOUT
end

Instance Method Details

#<=>(other) ⇒ Object 



140
141
142
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 140

def <=>(other)
  id <=> other.id
end

#after_sync ⇒ Object 



155
156
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 155

def after_sync
end

#current_generation ⇒ Object 



187
188
189
190
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 187

def current_generation
  return -1 unless exists? # must be different default than observed_generation
  @instance_data.dig("metadata", "generation")
end

#debug_message(cause = nil, info_hash = {}) ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 225

def debug_message(cause = nil, info_hash = {})
  helpful_info = []
  if cause == :gave_up
    debug_heading = ColorizedString.new("#{id}: GLOBAL WATCH TIMEOUT (#{info_hash[:timeout]} seconds)").yellow
    helpful_info << "If you expected it to take longer than #{info_hash[:timeout]} seconds for your deploy"\
    " to roll out, increase --max-watch-seconds."
  elsif deploy_failed?
    debug_heading = ColorizedString.new("#{id}: FAILED").red
    helpful_info << failure_message if failure_message.present?
  elsif deploy_timed_out?
    debug_heading = ColorizedString.new("#{id}: TIMED OUT (#{pretty_timeout_type})").yellow
    helpful_info << timeout_message if timeout_message.present?
  else
    # Arriving in debug_message when we neither failed nor timed out is very unexpected. Dump all available info.
    debug_heading = ColorizedString.new("#{id}: MONITORING ERROR").red
    helpful_info << failure_message if failure_message.present?
    helpful_info << timeout_message if timeout_message.present? && timeout_message != STANDARD_TIMEOUT_MESSAGE
  end

  final_status = "  - Final status: #{status}"
  final_status = "\n#{final_status}" if helpful_info.present? && !helpful_info.last.end_with?("\n")
  helpful_info.prepend(debug_heading)
  helpful_info << final_status

  if @debug_events.present?
    helpful_info << "  - Events (common success events excluded):"
    @debug_events.each do |identifier, event_hashes|
      event_hashes.each { |event| helpful_info << "      [#{identifier}]\t#{event}" }
    end
  elsif ENV[DISABLE_FETCHING_EVENT_INFO]
    helpful_info << "  - Events: #{DISABLED_EVENT_INFO_MESSAGE}"
  else
    helpful_info << "  - Events: #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
  end

  if print_debug_logs?
    if ENV[DISABLE_FETCHING_LOG_INFO]
      helpful_info << "  - Logs: #{DISABLED_LOG_INFO_MESSAGE}"
    elsif @debug_logs.blank?
      helpful_info << "  - Logs: #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
    else
      container_logs = @debug_logs.container_logs.sort_by { |c| c.lines.length }
      container_logs.each do |logs|
        if logs.empty?
          helpful_info << "  - Logs from container '#{logs.container_name}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
          next
        end

        if logs.lines.length == ContainerLogs::DEFAULT_LINE_LIMIT
          truncated = " (last #{ContainerLogs::DEFAULT_LINE_LIMIT} lines shown)"
        end
        helpful_info << "  - Logs from container '#{logs.container_name}'#{truncated}:"
        logs.lines.each do |line|
          helpful_info << "      #{line}"
        end
      end
    end
  end

  helpful_info.join("\n")
end

#deploy_failed? ⇒ Boolean

Returns:

  • (Boolean) 


166
167
168
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 166

def deploy_failed?
  false
end

#deploy_method ⇒ Object

Expected values: :apply, :replace, :replace_force



216
217
218
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 216

def deploy_method
  :apply
end

#deploy_started? ⇒ Boolean

Returns:

  • (Boolean) 


170
171
172
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 170

def deploy_started?
  @deploy_started_at.present?
end

#deploy_succeeded? ⇒ Boolean

Returns:

  • (Boolean) 


174
175
176
177
178
179
180
181
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 174

def deploy_succeeded?
  return false unless deploy_started?
  unless @success_assumption_warning_shown
    @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")
    @success_assumption_warning_shown = true
  end
  true
end

#deploy_timed_out? ⇒ Boolean

Returns:

  • (Boolean) 


210
211
212
213
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 210

def deploy_timed_out?
  return false unless deploy_started?
  !deploy_succeeded? && !deploy_failed? && (Time.now.utc - @deploy_started_at > timeout)
end

#disappeared? ⇒ Boolean

Returns:

  • (Boolean) 


162
163
164
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 162

def disappeared?
  @disappeared
end

#exists? ⇒ Boolean

Returns:

  • (Boolean) 


183
184
185
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 183

def exists?
  @instance_data.present?
end

#failure_message ⇒ Object 



310
311
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 310

def failure_message
end

#fetch_events(kubectl) ⇒ Object

Returns a hash in the following format: 

"pod/web-1" => [
  "Pulling: pulling image "hello-world:latest" (1 events)",
  "Pulled: Successfully pulled image "hello-world:latest" (1 events)"
]




294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 294

def fetch_events(kubectl)
  return {} unless exists?
  out, _err, st = kubectl.run("get", "events", "--output=go-template=#{Event.go_template_for(type, name)}",
    log_failure: false)
  return {} unless st.success?

  event_collector = Hash.new { |hash, key| hash[key] = [] }
  Event.extract_all_from_go_template_blob(out).each_with_object(event_collector) do |candidate, events|
    events[id] << candidate.to_s if candidate.seen_since?(@deploy_started_at - 5.seconds)
  end
end

#file_path ⇒ Object 



144
145
146
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 144

def file_path
  file.path
end

#global? ⇒ Boolean

Returns:

  • (Boolean) 


389
390
391
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 389

def global?
  self.class::GLOBAL
end

#id ⇒ Object 



136
137
138
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 136

def id
  "#{type}/#{name}"
end

#kubectl_output_is_sensitive? ⇒ Boolean

Returns:

  • (Boolean) 


325
326
327
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 325

def kubectl_output_is_sensitive?
  self.class::KUBECTL_OUTPUT_IS_SENSITIVE
end

#kubectl_resource_type ⇒ Object 



206
207
208
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 206

def kubectl_resource_type
  type
end

#observed_generation ⇒ Object 



192
193
194
195
196
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 192

def observed_generation
  return -2 unless exists?
  # populating this is a best practice, but not all controllers actually do it
  @instance_data.dig('status', 'observedGeneration')
end

#pretty_status ⇒ Object 



313
314
315
316
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 313

def pretty_status
  padding = " " * [50 - id.length, 1].max
  "#{id}#{padding}#{status}"
end

#pretty_timeout_type ⇒ Object 



83
84
85
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 83

def pretty_timeout_type
  "timeout: #{timeout}s"
end

#report_status_to_statsd(watch_time) ⇒ Object 



318
319
320
321
322
323
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 318

def report_status_to_statsd(watch_time)
  unless @statsd_report_done
    StatsD.distribution('resource.duration', watch_time, tags: statsd_tags)
    @statsd_report_done = true
  end
end

#status ⇒ Object 



198
199
200
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 198

def status
  exists? ? "Exists" : "Not Found"
end

#sync(cache) ⇒ Object 



148
149
150
151
152
153
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 148

def sync(cache)
  @instance_data = cache.get_instance(kubectl_resource_type, name, raise_if_not_found: true)
rescue KubernetesDeploy::Kubectl::ResourceNotFoundError
  @disappeared = true if deploy_started?
  @instance_data = {}
end

#sync_debug_info(kubectl) ⇒ Object 



220
221
222
223
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 220

def sync_debug_info(kubectl)
  @debug_events = fetch_events(kubectl) unless ENV[DISABLE_FETCHING_EVENT_INFO]
  @debug_logs = fetch_debug_logs if print_debug_logs? && !ENV[DISABLE_FETCHING_LOG_INFO]
end

#terminating? ⇒ Boolean

Returns:

  • (Boolean) 


158
159
160
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 158

def terminating?
  @instance_data.dig('metadata', 'deletionTimestamp').present?
end

#timeout ⇒ Object 



71
72
73
74
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 71

def timeout
  return timeout_override if timeout_override.present?
  self.class.timeout
end

#timeout_message ⇒ Object 



306
307
308
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 306

def timeout_message
  STANDARD_TIMEOUT_MESSAGE
end

#timeout_override ⇒ Object 



76
77
78
79
80
81
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 76

def timeout_override
  return @timeout_override if defined?(@timeout_override)
  @timeout_override = DurationParser.new(timeout_annotation).parse!.to_i
rescue DurationParser::ParsingError
  @timeout_override = nil
end

#to_kubeclient_resource ⇒ Object 



106
107
108
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 106

def to_kubeclient_resource
  Kubeclient::Resource.new(@definition)
end

#validate_definition(kubectl, selector: nil) ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 110

def validate_definition(kubectl, selector: nil)
  @validation_errors = []
  validate_selector(selector) if selector
  validate_timeout_annotation

  command = ["create", "-f", file_path, "--dry-run", "--output=name"]
  _, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: kubectl_output_is_sensitive?)
  return true if st.success?
  if kubectl_output_is_sensitive?
    @validation_errors << <<-EOS
      Validation for #{id} failed. Detailed information is unavailable as the raw error may contain sensitive data.
    EOS
  else
    @validation_errors << err
  end
  false
end

#validation_error_msg ⇒ Object 



128
129
130
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 128

def validation_error_msg
  @validation_errors.join("\n")
end

#validation_failed? ⇒ Boolean

Returns:

  • (Boolean) 


132
133
134
 
# File 'lib/kubernetes-deploy/kubernetes_resource.rb', line 132

def validation_failed?
  @validation_errors.present?
end