Class: Demon::Sidekiq

Inherits:
Base
  • Object
show all
Defined in:
lib/demon/sidekiq.rb

Constant Summary collapse

SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS =

By default Sidekiq does a heartbeat check every 5 seconds. If the processes misses 20 heartbeat checks, we consider it dead and kill the process.

5.seconds * 20
SIDEKIQ_RSS_MEMORY_CHECK_INTERVAL_SECONDS =
30.minutes
DEFAULT_MAX_ALLOWED_SIDEKIQ_RSS_MEGABYTES =
500

Constants inherited from Base

Base::HOSTNAME

Instance Attribute Summary

Attributes inherited from Base

#index, #parent_pid, #pid, #started, #stop_timeout

Class Method Summary collapse

Methods inherited from Base

#alive?, alive?, #already_running?, demons, #ensure_running, ensure_running, #initialize, kill, #kill, #log, #pid_file, reset_demons, restart, #restart, #run, set_demons, #set_pid, start, #start, stop, #stop, #stop_signal

Constructor Details

This class inherits a constructor from Demon::Base

Class Method Details

.after_fork(&blk) ⇒ Object



10
11
12
# File 'lib/demon/sidekiq.rb', line 10

def self.after_fork(&blk)
  blk ? (@blk = blk) : @blk
end

.heartbeat_checkObject



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# File 'lib/demon/sidekiq.rb', line 18

def self.heartbeat_check
  sidekiq_processes_for_current_hostname = {}

  Sidekiq::ProcessSet.new.each do |process|
    if process["hostname"] == HOSTNAME
      sidekiq_processes_for_current_hostname[process["pid"]] = process
    end
  end

  Demon::Sidekiq.demons.values.each do |daemon|
    next if !daemon.already_running?

    running_sidekiq_process = sidekiq_processes_for_current_hostname[daemon.pid]

    if !running_sidekiq_process ||
         (Time.now.to_i - running_sidekiq_process["beat"]) >
           SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS
      Rails.logger.warn("Sidekiq heartbeat test failed for #{daemon.pid}, restarting")
      daemon.restart
    end
  end
end

.max_allowed_sidekiq_rss_bytesObject



69
70
71
# File 'lib/demon/sidekiq.rb', line 69

def self.max_allowed_sidekiq_rss_bytes
  [ENV["UNICORN_SIDEKIQ_MAX_RSS"].to_i, DEFAULT_MAX_ALLOWED_SIDEKIQ_RSS_MEGABYTES].max.megabytes
end

.prefixObject



6
7
8
# File 'lib/demon/sidekiq.rb', line 6

def self.prefix
  "sidekiq"
end

.rss_memory_checkObject



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# File 'lib/demon/sidekiq.rb', line 43

def self.rss_memory_check
  if defined?(@@last_sidekiq_rss_memory_check) && @@last_sidekiq_rss_memory_check &&
       @@last_sidekiq_rss_memory_check > Time.now.to_i - SIDEKIQ_RSS_MEMORY_CHECK_INTERVAL_SECONDS
    return @@last_sidekiq_rss_memory_check
  end

  Demon::Sidekiq.demons.values.each do |daemon|
    next if !daemon.already_running?

    daemon_rss_bytes = (`ps -o rss= -p #{daemon.pid}`.chomp.to_i || 0) * 1024

    if daemon_rss_bytes > max_allowed_sidekiq_rss_bytes
      Rails.logger.warn(
        "Sidekiq is consuming too much memory (using: %0.2fM) for '%s', restarting" %
          [(daemon_rss_bytes.to_f / 1.megabyte), HOSTNAME],
      )

      daemon.restart
    end
  end

  @@last_sidekiq_rss_memory_check = Time.now.to_i
end