Class: Gitlab::SidekiqCluster::CLI

Inherits:
Object
  • Object
show all
Defined in:
lib/gitlab/sidekiq_cluster/cli.rb

Constant Summary collapse

CHECK_TERMINATE_INTERVAL_SECONDS = 
1
DEFAULT_SOFT_TIMEOUT_SECONDS =

How long to wait when asking for a clean termination. It maps the Sidekiq default timeout: github.com/mperham/sidekiq/wiki/Signals#term

This value is passed to Sidekiq's `-t` if none is given through arguments.

25
DEFAULT_HARD_TIMEOUT_SECONDS =

After surpassing the soft timeout.

5
CommandError = 
Class.new(StandardError)

Instance Method Summary collapse

Constructor Details

#initialize(log_output = STDERR) ⇒ CLI

Returns a new instance of CLI.


25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 25

def initialize(log_output = STDERR)
  require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter'

  # As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency
  @max_concurrency = 50
  @min_concurrency = 0
  @environment = ENV['RAILS_ENV'] || 'development'
  @pid = nil
  @interval = 5
  @alive = true
  @processes = []
  @logger = Logger.new(log_output)
  @logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new
  @rails_path = Dir.pwd
  @dryrun = false
end

Instance Method Details

#continue_waiting?(deadline) ⇒ Boolean

Returns:

  • (Boolean) 

113
114
115
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 113

def continue_waiting?(deadline)
  SidekiqCluster.any_alive?(@processes) && monotonic_time < deadline
end

#hard_stop_stuck_pids ⇒ Object 


117
118
119
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 117

def hard_stop_stuck_pids
  SidekiqCluster.signal_processes(SidekiqCluster.pids_alive(@processes), "-KILL")
end

#hard_timeout_seconds ⇒ Object

The amount of time it'll wait for killing the alive Sidekiq processes.


105
106
107
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 105

def hard_timeout_seconds
  soft_timeout_seconds + DEFAULT_HARD_TIMEOUT_SECONDS
end

#monotonic_time ⇒ Object 


109
110
111
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 109

def monotonic_time
  Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
end

#option_parser ⇒ Object 


155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 155

def option_parser
  OptionParser.new do |opt|
    opt.banner = "#{File.basename(__FILE__)} [QUEUE,QUEUE] [QUEUE] ... [OPTIONS]"

    opt.separator "\nOptions:\n"

    opt.on('-h', '--help', 'Shows this help message') do
      abort opt.to_s
    end

    opt.on('-m', '--max-concurrency INT', 'Maximum threads to use with Sidekiq (default: 50, 0 to disable)') do |int|
      @max_concurrency = int.to_i
    end

    opt.on('--min-concurrency INT', 'Minimum threads to use with Sidekiq (default: 0)') do |int|
      @min_concurrency = int.to_i
    end

    opt.on('-e', '--environment ENV', 'The application environment') do |env|
      @environment = env
    end

    opt.on('-P', '--pidfile PATH', 'Path to the PID file') do |pid|
      @pid = pid
    end

    opt.on('-r', '--require PATH', 'Location of the Rails application') do |path|
      @rails_path = path
    end

    opt.on('--experimental-queue-selector', 'EXPERIMENTAL: Run workers based on the provided selector') do |experimental_queue_selector|
      @experimental_queue_selector = experimental_queue_selector
    end

    opt.on('-n', '--negate', 'Run workers for all queues in sidekiq_queues.yml except the given ones') do
      @negate_queues = true
    end

    opt.on('-i', '--interval INT', 'The number of seconds to wait between worker checks') do |int|
      @interval = int.to_i
    end

    opt.on('-t', '--timeout INT', 'Graceful timeout for all running processes') do |timeout|
      @soft_timeout_seconds = timeout.to_i
    end

    opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int|
      @dryrun = true
    end
  end
end

#run(argv = ARGV) ⇒ Object 


42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 42

def run(argv = ARGV)
  if argv.empty?
    raise CommandError,
      'You must specify at least one queue to start a worker for'
  end

  option_parser.parse!(argv)

  all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path)
  queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path)

  queue_groups = argv.map do |queues|
    next queue_names if queues == '*'

    # When using the experimental queue query syntax, we treat
    # each queue group as a worker attribute query, and resolve
    # the queues for the queue group using this query.
    if @experimental_queue_selector
      SidekiqConfig::CliMethods.query_workers(queues, all_queues)
    else
      SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names)
    end
  end

  if @negate_queues
    queue_groups.map! { |queues| queue_names - queues }
  end

  if queue_groups.all?(&:empty?)
    raise CommandError,
      'No queues found, you must select at least one queue'
  end

  unless @dryrun
    @logger.info("Starting cluster with #{queue_groups.length} processes")
  end

  @processes = SidekiqCluster.start(
    queue_groups,
    env: @environment,
    directory: @rails_path,
    max_concurrency: @max_concurrency,
    min_concurrency: @min_concurrency,
    dryrun: @dryrun,
    timeout: soft_timeout_seconds
  )

  return if @dryrun

  write_pid
  trap_signals
  start_loop
end

#soft_timeout_seconds ⇒ Object 


100
101
102
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 100

def soft_timeout_seconds
  @soft_timeout_seconds || DEFAULT_SOFT_TIMEOUT_SECONDS
end

#start_loop ⇒ Object 


140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 140

def start_loop
  while @alive
    sleep(@interval)

    unless SidekiqCluster.all_alive?(@processes)
      # If a child process died we'll just terminate the whole cluster. It's up to
      # runit and such to then restart the cluster.
      @logger.info('A worker terminated, shutting down the cluster')

      SidekiqCluster.signal_processes(@processes, :TERM)
      break
    end
  end
end

#trap_signals ⇒ Object 


128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 128

def trap_signals
  SidekiqCluster.trap_terminate do |signal|
    @alive = false
    SidekiqCluster.signal_processes(@processes, signal)
    wait_for_termination
  end

  SidekiqCluster.trap_forward do |signal|
    SidekiqCluster.signal_processes(@processes, signal)
  end
end

#wait_for_termination ⇒ Object 


121
122
123
124
125
126
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 121

def wait_for_termination
  deadline = monotonic_time + hard_timeout_seconds
  sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline)

  hard_stop_stuck_pids
end

#write_pid ⇒ Object 


96
97
98
 
# File 'lib/gitlab/sidekiq_cluster/cli.rb', line 96

def write_pid
  SidekiqCluster.write_pid(@pid) if @pid
end