Module: SqsPoller

Defined in:
lib/sqspoller/sqs_poller.rb,
lib/sqspoller.rb,
lib/sqspoller/version.rb,
lib/sqspoller/common/utils.rb,
lib/sqspoller/logger/logger.rb,
lib/sqspoller/poll/queue_poller.rb,
lib/sqspoller/common/ring_buffer.rb,
lib/sqspoller/process/task_worker.rb,
lib/sqspoller/metrics/log_reporter.rb,
lib/sqspoller/poll/queue_controller.rb,
lib/sqspoller/process/task_finalizer.rb,
lib/sqspoller/process/message_handler.rb,
lib/sqspoller/process/worker_controller.rb,
lib/sqspoller/metrics/sqs_poller_metrics.rb,
lib/sqspoller/metrics/queue_stats_reporter.rb

Overview

This reports the Aws::SQS::QueuePoller stats for each queue and each worker in the log.

Defined Under Namespace

Modules: Common, Logger, Metrics, Poller, Process

Constant Summary collapse

VERSION = 
"2.0.0"

Class Method Summary collapse

Class Method Details

.daemonize(filename) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/sqspoller/sqs_poller.rb', line 26

def daemonize(filename)
  raise 'Must run as root' if Process.euid != 0

  raise 'First fork failed' if (pid = fork) == -1
  exit unless pid.nil?

  Process.setsid
  raise 'Second fork failed' if (pid = fork) == -1
  exit unless pid.nil?
  puts "Daemon pid: #{Process.pid}" # Or save it somewhere, etc.

  Dir.chdir '/'
  File.umask 0000

  STDIN.reopen filename
  STDOUT.reopen '/dev/null', 'a'
  STDERR.reopen STDOUT
end

.get_total_poller_threads(queues_config) ⇒ Object 



108
109
110
111
112
113
114
 
# File 'lib/sqspoller/sqs_poller.rb', line 108

def get_total_poller_threads(queues_config)
  total_poller_threads = 0
  queues_config.keys.each { |queue|
    total_poller_threads += queues_config[queue][:polling_threads]
  }
  total_poller_threads
end

.get_total_worker_threads(worker_configuration, total_poller_threads) ⇒ Object 



122
123
124
125
126
 
# File 'lib/sqspoller/sqs_poller.rb', line 122

def get_total_worker_threads(worker_configuration, total_poller_threads)
  worker_thread_count = worker_configuration[:concurrency]
  worker_thread_count = total_poller_threads if worker_thread_count.nil?
  worker_thread_count
end

.get_waiting_tasks_ratio(worker_configuration) ⇒ Object 



116
117
118
119
120
 
# File 'lib/sqspoller/sqs_poller.rb', line 116

def get_waiting_tasks_ratio(worker_configuration)
  waiting_tasks_ratio = worker_configuration[:waiting_tasks_ratio]
  waiting_tasks_ratio = 1 if waiting_tasks_ratio.nil?
  waiting_tasks_ratio
end

.start_poller(filename, queue_config_name, access_key_id, secret_access_key, region, log_filename = nil) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/sqspoller/sqs_poller.rb', line 85

def start_poller(filename, queue_config_name, access_key_id, secret_access_key, region, log_filename = nil)
  begin
    puts "Starting poller"
    config = YAML.load(ERB.new(IO.read(filename)).result)
    config = sym(config)

    if log_filename.nil? || log_filename.empty?
      puts "Did not receive log file name"
      fork do
        Process.daemon
        start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, STDOUT, ::Logger::DEBUG
      end
    else
      puts "Did receive log file name #{log_filename}"
      #daemonize log_filename
      puts "Daemonize log file name #{log_filename}"
      start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, STDOUT, ::Logger::INFO
    end
  rescue Exception => e
    puts "#{e}"
  end
end

.start_poller_with_config(config, queue_config_name, access_key_id, secret_access_key, region, logger_file, log_level = ::Logger::ERROR) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/sqspoller/sqs_poller.rb', line 45

def start_poller_with_config(config, queue_config_name, access_key_id, secret_access_key, region, logger_file, log_level = ::Logger::ERROR)
  SqsPoller::Logger.set_log_level(log_level)
  SqsPoller::Logger.set_logger_file(logger_file)

  ::NewRelic::Agent.manual_start if ENV['START_NEW_RELIC_AGENT']
  SqsPoller::Metrics.start_metrics_agent if ENV['START_LOCAL_METRICS_AGENT']
  @logger = SqsPoller::Logger.get_new_logger("SqsPoller")
  @logger.info "Started poller method"
  queues_config = config[queue_config_name] || config[queue_config_name.to_sym]
  total_poller_threads = get_total_poller_threads(queues_config)
  worker_configuration = config[:worker_configuration]
  total_worker_threads = get_total_worker_threads(worker_configuration, total_poller_threads)
  waiting_tasks_ratio = get_waiting_tasks_ratio(worker_configuration)
  aws_config = {
    :access_key_id => access_key_id,
    :secret_access_key => secret_access_key,
    :region => region
  }

  task_queue = SizedQueue.new(total_worker_threads * waiting_tasks_ratio)

  qc = SqsPoller::Poller::QueueController.start queues_config, task_queue, aws_config
  unless qc.started?
    @logger.error("Unable to start Queue Pollers.")
    return
  end

  worker_task = worker_configuration[:worker_class].split('::').inject(Object) { |o, c| o.const_get c }.new(worker_configuration)

  wc = SqsPoller::Process::WorkerController.start total_worker_threads, task_queue, worker_task

  unless wc.started?
    @logger.error("Unable to start Workers.")
    return
  end

  wait

end

.sym(map) ⇒ Object 



19
20
21
22
23
24
 
# File 'lib/sqspoller/sqs_poller.rb', line 19

def sym(map)
  if map.class == Hash
    map = map.inject({}) { |memo, (k, v)| memo[k.to_sym] = sym(v); memo }
  end
  map
end

.timeout ⇒ Object 



128
129
130
131
132
 
# File 'lib/sqspoller/sqs_poller.rb', line 128

def timeout
  n_bytes = [42].pack('i').size
  n_bits = n_bytes * 8
  2 ** (n_bits - 2) - 1
end