Class: HybridPlatformsConductor::Deployer

Inherits:
Object
  • Object
show all
Includes:
LoggerHelpers
Defined in:
lib/hybrid_platforms_conductor/deployer.rb

Overview

Gives ways to deploy on several nodes

Defined Under Namespace

Modules: ConfigDSLExtension

Constant Summary collapse

PACKAGING_FUTEX_FILE =

String: File used as a Futex for packaging

"#{Dir.tmpdir}/hpc_packaging"

Constants included from LoggerHelpers

LoggerHelpers::LEVELS_MODIFIERS, LoggerHelpers::LEVELS_TO_STDERR

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from LoggerHelpers

#err, #init_loggers, #log_component=, #log_debug?, #log_level=, #out, #section, #set_loggers_format, #stderr_device, #stderr_device=, #stderr_displayed?, #stdout_device, #stdout_device=, #stdout_displayed?, #stdouts_to_s, #with_progress_bar

Constructor Details

#initialize(logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR), config: Config.new, cmd_runner: CmdRunner.new, nodes_handler: NodesHandler.new, actions_executor: ActionsExecutor.new, services_handler: ServicesHandler.new) ⇒ Deployer

Constructor

Parameters

  • logger (Logger): Logger to be used [default: Logger.new(STDOUT)]

  • logger_stderr (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]

  • config (Config): Config to be used. [default: Config.new]

  • cmd_runner (CmdRunner): Command executor to be used. [default: CmdRunner.new]

  • nodes_handler (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]

  • actions_executor (ActionsExecutor): Actions Executor to be used. [default: ActionsExecutor.new]

  • services_handler (ServicesHandler): Services Handler to be used. [default: ServicesHandler.new]



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 98

def initialize(
  logger: Logger.new(STDOUT),
  logger_stderr: Logger.new(STDERR),
  config: Config.new,
  cmd_runner: CmdRunner.new,
  nodes_handler: NodesHandler.new,
  actions_executor: ActionsExecutor.new,
  services_handler: ServicesHandler.new
)
  init_loggers(logger, logger_stderr)
  @config = config
  @cmd_runner = cmd_runner
  @nodes_handler = nodes_handler
  @actions_executor = actions_executor
  @services_handler = services_handler
  @secrets = []
  @provisioners = Plugins.new(:provisioner, logger: @logger, logger_stderr: @logger_stderr)
  @log_plugins = Plugins.new(
    :log,
    logger: @logger,
    logger_stderr: @logger_stderr,
    init_plugin: proc do |plugin_class|
      plugin_class.new(
        logger: @logger,
        logger_stderr: @logger_stderr,
        config: @config,
        nodes_handler: @nodes_handler,
        actions_executor: @actions_executor
      )
    end
  )
  # Default values
  @use_why_run = false
  @timeout = nil
  @concurrent_execution = false
  @local_environment = false
  @nbr_retries_on_error = 0
end

Instance Attribute Details

#concurrent_execution ⇒ Object

Concurrent execution of the deployment? [default = false]

Boolean



74
75
76
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 74

def concurrent_execution
  @concurrent_execution
end

#local_environment ⇒ Object

Are we deploying in a local environment?

Boolean



82
83
84
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 82

def local_environment
  @local_environment
end

#nbr_retries_on_error ⇒ Object

Number of retries to do in case of non-deterministic errors during deployment

Integer



86
87
88
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 86

def nbr_retries_on_error
  @nbr_retries_on_error
end

#secrets ⇒ Object

The list of JSON secrets

Array<Hash>



78
79
80
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 78

def secrets
  @secrets
end

#timeout ⇒ Object

Timeout (in seconds) to be used for each deployment, or nil for no timeout [default = nil]

Integer or nil



70
71
72
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 70

def timeout
  @timeout
end

#use_why_run ⇒ Object

Do we use why-run mode while deploying? [default = false]

Boolean



66
67
68
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 66

def use_why_run
  @use_why_run
end

Instance Method Details

#deploy_on(*nodes_selectors) ⇒ Object

Deploy on a given list of nodes selectors. The workflow is the following:

  1. Package the services to be deployed, considering the nodes, services and context (options, secrets, environment…)

  2. Deploy on the nodes (once per node to be deployed)

  3. Save deployment logs (in case of real deployment)

Parameters

  • nodes_selectors (Array<Object>): The list of nodes selectors we will deploy to.

Result

  • Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node that has been deployed.



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 203

def deploy_on(*nodes_selectors)
  # Get the sorted list of services to be deployed, per node
  # Hash<String, Array<String> >
  services_to_deploy = Hash[@nodes_handler.select_nodes(nodes_selectors.flatten).map do |node|
    [node, @nodes_handler.get_services_of(node)]
  end]

  # Get the secrets to be deployed
  secrets = {}
  @secrets.each do |secret_json|
    secrets.merge!(secret_json) do |key, value1, value2|
      raise "Secret #{key} has conflicting values between different secret JSON files." if value1 != value2
      value1
    end
  end

  # Check that we are allowed to deploy
  unless @use_why_run
    reason_for_interdiction = @services_handler.deploy_allowed?(
      services: services_to_deploy,
      secrets: secrets,
      local_environment: @local_environment
    )
    raise "Deployment not allowed: #{reason_for_interdiction}" unless reason_for_interdiction.nil?
  end

  # Package the deployment
  # Protect packaging by a Futex
  Futex.new(PACKAGING_FUTEX_FILE, timeout: @config.packaging_timeout_secs).open do
    section 'Packaging deployment' do
      @services_handler.package(
        services: services_to_deploy,
        secrets: secrets,
        local_environment: @local_environment
      )
    end
  end

  # Prepare the deployment as a whole, before getting individual deployment actions.
  # Do this after packaging, this way we ensure that services packaging cannot depend on the way deployment will be performed.
  @services_handler.prepare_for_deploy(
    services: services_to_deploy,
    secrets: secrets,
    local_environment: @local_environment,
    why_run: @use_why_run
  )

  # Launch deployment processes
  results = {}

  section "#{@use_why_run ? 'Checking' : 'Deploying'} on #{services_to_deploy.keys.size} nodes" do
    # Prepare all the control masters here, as they will be reused for the whole process, including mutexes, deployment and logs saving
    @actions_executor.with_connections_prepared_to(services_to_deploy.keys, no_exception: true) do

      nbr_retries = @nbr_retries_on_error
      remaining_nodes_to_deploy = services_to_deploy.keys
      while nbr_retries >= 0 && !remaining_nodes_to_deploy.empty?
        last_deploy_results = deploy(services_to_deploy.slice(*remaining_nodes_to_deploy))
        if nbr_retries > 0
          # Check if we need to retry deployment on some nodes
          # Only parse the last deployment attempt logs
          retriable_nodes = Hash[
            remaining_nodes_to_deploy.
              map do |node|
                exit_status, stdout, stderr = last_deploy_results[node]
                if exit_status == 0
                  nil
                else
                  retriable_errors = retriable_errors_from(node, exit_status, stdout, stderr)
                  if retriable_errors.empty?
                    nil
                  else
                    # Log the issue in the stderr of the deployment
                    stderr << "!!! #{retriable_errors.size} retriable errors detected in this deployment:\n#{retriable_errors.map { |error| "* #{error}" }.join("\n")}\n"
                    [node, retriable_errors]
                  end
                end
              end.
              compact
          ]
          unless retriable_nodes.empty?
            log_warn <<~EOS.strip
              Retry deployment for #{retriable_nodes.size} nodes as they got non-deterministic errors (#{nbr_retries} retries remaining):
              #{retriable_nodes.map { |node, retriable_errors| "  * #{node}:\n#{retriable_errors.map { |error| "    - #{error}" }.join("\n")}" }.join("\n")}
            EOS
          end
          remaining_nodes_to_deploy = retriable_nodes.keys
        end
        # Merge deployment results
        results.merge!(last_deploy_results) do |node, (exit_status_1, stdout_1, stderr_1), (exit_status_2, stdout_2, stderr_2)|
          [
            exit_status_2,
            <<~EOS,
            <<~EOS
              #{stderr_1}
              !!! Retry deployment due to non-deterministic error (#{nbr_retries} remaining attempts)...
              #{stderr_2}
            EOS
          ]
        end
        nbr_retries -= 1
      end

    end
  end
  results
end

#deployment_info_from(*nodes) ⇒ Object

Get deployment info from a list of nodes selectors

Parameters

  • nodes (Array<String>): Nodes to get info from

Result

  • Hash<String, Hash<Symbol,Object>: The deployed info, per node name.

    • error (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]

    • services (Array<String>): List of services deployed on the node

    • deployment_info (Hash<Symbol,Object>): Deployment metadata

    • exit_status (Integer or Symbol): Deployment exit status

    • stdout (String): Deployment stdout

    • stderr (String): Deployment stderr



396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 396

def deployment_info_from(*nodes)
  nodes = nodes.flatten
  @actions_executor.max_threads = 64
  read_actions_results = @actions_executor.execute_actions(
    Hash[nodes.map do |node|
      master_log_plugin = @log_plugins[log_plugins_for(node).first]
      master_log_plugin.respond_to?(:actions_to_read_logs) ? [node, master_log_plugin.actions_to_read_logs(node)] : nil
    end.compact],
    log_to_stdout: false,
    concurrent: true,
    timeout: 10,
    progress_name: 'Read deployment logs'
  )
  Hash[nodes.map do |node|
    [
      node,
      @log_plugins[log_plugins_for(node).first].logs_for(node, *(read_actions_results[node] || [nil, nil, nil]))
    ]
  end]
end

#options_parse(options_parser, parallel_switch: true, why_run_switch: false, timeout_options: true) ⇒ Object

Complete an option parser with options meant to control this Deployer

Parameters

  • options_parser (OptionParser): The option parser to complete

  • parallel_switch (Boolean): Do we allow parallel execution to be switched? [default = true]

  • why_run_switch (Boolean): Do we allow the why-run mode to be switched? [default = false]

  • timeout_options (Boolean): Do we allow timeout options? [default = true]



144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 144

def options_parse(options_parser, parallel_switch: true, why_run_switch: false, timeout_options: true)
  options_parser.separator ''
  options_parser.separator 'Deployer options:'
  options_parser.on(
    '-e', '--secrets SECRETS_LOCATION',
    'Specify a secrets location. Can be specified several times. Location can be:',
    '* Local path to a JSON file',
    '* URL of the form http[s]://<url>:<secret_id> to get a secret JSON file from a Thycotic Secret Server at the given URL.'
  ) do |secrets_location|
    @secrets << JSON.parse(
      if secrets_location =~ /^(https?:\/\/.+):(\d+)$/
        url = $1
        secret_id = $2
        secret = nil
        Thycotic.with_thycotic(url, @logger, @logger_stderr) do |thycotic|
          secret_file_item_id = thycotic.get_secret(secret_id).dig(:secret, :items, :secret_item, :id)
          raise "Unable to fetch secret file ID #{secrets_location}" if secret_file_item_id.nil?
          secret = thycotic.download_file_attachment_by_item_id(secret_id, secret_file_item_id)
          raise "Unable to fetch secret file attachment from #{secrets_location}" if secret.nil?
        end
        secret
      else
        raise "Missing secret file: #{secrets_location}" unless File.exist?(secrets_location)
        File.read(secrets_location)
      end
    )
  end
  options_parser.on('-p', '--parallel', 'Execute the commands in parallel (put the standard output in files <hybrid-platforms-dir>/run_logs/*.stdout)') do
    @concurrent_execution = true
  end if parallel_switch
  options_parser.on('-t', '--timeout SECS', "Timeout in seconds to wait for each chef run. Only used in why-run mode. (defaults to #{@timeout.nil? ? 'no timeout' : @timeout})") do |nbr_secs|
    @timeout = nbr_secs.to_i
  end if timeout_options
  options_parser.on('-W', '--why-run', 'Use the why-run mode to see what would be the result of the deploy instead of deploying it for real.') do
    @use_why_run = true
  end if why_run_switch
  options_parser.on('--retries-on-error NBR', "Number of retries in case of non-deterministic errors (defaults to #{@nbr_retries_on_error})") do |nbr_retries|
    @nbr_retries_on_error = nbr_retries.to_i
  end
end

#parse_deploy_output(node, stdout, stderr) ⇒ Object

Parse stdout and stderr of a given deploy run and get the list of tasks with their status

Parameters

  • node (String): Node for which this deploy run has been done.

  • stdout (String): stdout to be parsed.

  • stderr (String): stderr to be parsed.

Result

  • Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:

    • name (String): Task name

    • status (Symbol): Task status. Should be on of:

      • :changed: The task has been changed

      • :identical: The task has not been changed

    • diffs (String): Differences, if any



430
431
432
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 430

def parse_deploy_output(node, stdout, stderr)
  @services_handler.parse_deploy_output(stdout, stderr).map { |deploy_info| deploy_info[:tasks] }.flatten
end

#validate_params ⇒ Object

Validate that parsed parameters are valid



186
187
188
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 186

def validate_params
  raise 'Can\'t have a timeout unless why-run mode. Please don\'t use --timeout without --why-run.' if !@timeout.nil? && !@use_why_run
end

#with_test_provisioned_instance(provisioner_id, node, environment:, reuse_instance: false) ⇒ Object

Provision a test instance for a given node.

Parameters

  • provisioner_id (Symbol): The provisioner ID to be used

  • node (String): The node for which we want the image

  • environment (String): An ID to differentiate different running instances for the same node

  • reuse_instance (Boolean): Do we reuse an eventual existing instance? [default: false]

  • Proc: Code called when the container is ready. The container will be stopped at the end of execution.

    • Parameters

      • deployer (Deployer): A new Deployer configured to override access to the node through the Docker container

      • instance (Provisioner): The provisioned instance



327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 327

def with_test_provisioned_instance(provisioner_id, node, environment:, reuse_instance: false)
  # Add the user to the environment to better track belongings on shared provisioners
  environment = "#{@cmd_runner.whoami}_#{environment}"
  # Add PID, TID and a random number to the ID to make sure other containers used by other runs are not being reused.
  environment << "_#{Process.pid}_#{Thread.current.object_id}_#{SecureRandom.hex(8)}" unless reuse_instance
  # Create different NodesHandler and Deployer to handle this Docker container in place of the real node.
  sub_logger, sub_logger_stderr =
    if log_debug?
      [@logger, @logger_stderr]
    else
      [Logger.new(StringIO.new, level: :info), Logger.new(StringIO.new, level: :info)]
    end
  begin
    sub_executable = Executable.new(logger: sub_logger, logger_stderr: sub_logger_stderr)
    instance = @provisioners[provisioner_id].new(
      node,
      environment: environment,
      logger: @logger,
      logger_stderr: @logger_stderr,
      config: sub_executable.config,
      cmd_runner: @cmd_runner,
      # Here we use the NodesHandler that will be bound to the sub-Deployer only, as the node's metadata might be modified by the Provisioner.
      nodes_handler: sub_executable.nodes_handler,
      actions_executor: @actions_executor
    )
    instance.with_running_instance(stop_on_exit: true, destroy_on_exit: !reuse_instance, port: 22) do
      # Test-provisioned nodes have SSH Session Exec capabilities and are not local
      sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, true
      sub_executable.nodes_handler.override_metadata_of node, :local_node, false
      # Test-provisioned nodes use default sudo
      sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
        {
          nodes_selectors_stack: sudo_proc_info[:nodes_selectors_stack].map do |nodes_selector|
            @nodes_handler.select_nodes(nodes_selector).select { |selected_node| selected_node != node }
          end,
          sudo_proc: sudo_proc_info[:sudo_proc]
        }
      end)
      actions_executor = sub_executable.actions_executor
      deployer = sub_executable.deployer
      # Setup test environment for this container
      actions_executor.connector(:ssh).ssh_user = 'root'
      actions_executor.connector(:ssh).passwords[node] = 'root_pwd'
      deployer.local_environment = true
      # Ignore secrets that might have been given: in Docker containers we always use dummy secrets
      dummy_secrets_file = "#{@config.hybrid_platforms_dir}/dummy_secrets.json"
      deployer.secrets = File.exist?(dummy_secrets_file) ? [JSON.parse(File.read(dummy_secrets_file))] : []
      yield deployer, instance
    end
  rescue
    # Make sure Docker logs are being output to better investigate errors if we were not already outputing them in debug mode
    stdouts = sub_executable.stdouts_to_s
    log_error "[ #{node}/#{environment} ] - Encountered unhandled exception #{$!}\n#{$!.backtrace.join("\n")}\n-----\n#{stdouts}" unless stdouts.nil?
    raise
  end
end