Class: Jiggler::Config

Inherits:
Object
  • Object
show all
Extended by:
Forwardable
Defined in:
lib/jiggler/config.rb

Constant Summary collapse

DEFAULT_QUEUE = 
'default'
QUEUE_PREFIX = 
'jiggler:list:'
SERVER_PREFIX = 
'jiggler:svr:'
RETRIES_SET = 
'jiggler:set:retries'
SCHEDULED_SET = 
'jiggler:set:scheduled'
DEAD_SET = 
'jiggler:set:dead'
PROCESSED_COUNTER = 
'jiggler:stats:processed_counter'
FAILURES_COUNTER = 
'jiggler:stats:failures_counter'
DEFAULTS = 
{
  require: nil,
  environment: 'development',
  concurrency: 10,
  timeout: 25,
  max_dead_jobs: 10_000,
  stats_interval: 10,
  poller_enabled: true,
  poll_interval: 5,
  # used in scheduled/requeuer
  in_process_interval: 120,
  dead_timeout: 180 * 24 * 60 * 60, # 6 months in seconds
  mode: :at_least_once,
  # client settings
  client_concurrency: 10,
  client_redis_pool: nil,
  fetchers_concurrency: 1,
  client_async: false
}

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ Config

Returns a new instance of Config.



39
40
41
42
43
44
 
# File 'lib/jiggler/config.rb', line 39

def initialize(options = {})
  @options = DEFAULTS.merge(options)
  @options[:redis_url] = ENV['REDIS_URL'] if @options[:redis_url].nil? && ENV['REDIS_URL']
  @options[:queues] ||= [DEFAULT_QUEUE]
  @directory = {}
end

Instance Method Details

#at_least_once? ⇒ Boolean

Returns:

  • (Boolean) 


87
88
89
 
# File 'lib/jiggler/config.rb', line 87

def at_least_once?
  @options[:mode] == :at_least_once
end

#cleaner ⇒ Object 



191
192
193
 
# File 'lib/jiggler/config.rb', line 191

def cleaner
  @cleaner ||= Jiggler::Cleaner.new(self)
end

#client_redis_options ⇒ Object 



164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/jiggler/config.rb', line 164

def client_redis_options
  @client_redis_options ||= begin
    opts = @options.slice(
      :redis_url,
      :client_redis_pool
    )

    opts[:concurrency] = @options[:client_concurrency]
    opts[:async] = @options[:client_async]
    opts
  end
end

#client_redis_pool ⇒ Object 



181
182
183
184
185
 
# File 'lib/jiggler/config.rb', line 181

def client_redis_pool
  @client_redis_pool ||= begin
    @options[:client_redis_pool] || Jiggler::RedisStore.new(client_redis_options).pool
  end
end

#client_redis_pool=(new_pool) ⇒ Object 



187
188
189
 
# File 'lib/jiggler/config.rb', line 187

def client_redis_pool=(new_pool)
  @client_redis_pool = new_pool
end

#dead_set ⇒ Object 



58
59
60
 
# File 'lib/jiggler/config.rb', line 58

def dead_set
  DEAD_SET
end

#default_queue ⇒ Object 



62
63
64
 
# File 'lib/jiggler/config.rb', line 62

def default_queue
  DEFAULT_QUEUE
end

#failures_counter ⇒ Object 



75
76
77
 
# File 'lib/jiggler/config.rb', line 75

def failures_counter
  FAILURES_COUNTER
end

#logger ⇒ Object 



203
204
205
 
# File 'lib/jiggler/config.rb', line 203

def logger
  @logger ||= ::Logger.new(STDOUT, level: :info)
end

#logger=(new_logger) ⇒ Object 



199
200
201
 
# File 'lib/jiggler/config.rb', line 199

def logger=(new_logger)
  @logger = new_logger
end

#process_scan_key ⇒ Object 



79
80
81
 
# File 'lib/jiggler/config.rb', line 79

def process_scan_key
  @process_scan_key ||= "#{server_prefix}*"
end

#processed_counter ⇒ Object 



71
72
73
 
# File 'lib/jiggler/config.rb', line 71

def processed_counter
  PROCESSED_COUNTER
end

#queue_prefix ⇒ Object 



46
47
48
 
# File 'lib/jiggler/config.rb', line 46

def queue_prefix
  QUEUE_PREFIX
end

#queue_scan_key ⇒ Object 



83
84
85
 
# File 'lib/jiggler/config.rb', line 83

def queue_scan_key
  @queue_scan_key ||= "#{queue_prefix}*"
end

#queues_data ⇒ Object 



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/jiggler/config.rb', line 91

def queues_data
  @queues_data ||= begin
    queues = {}

    @options[:queues].each do |queue|
      name, priority = queue
      # by default all queues have the same priority
      priority ||= 0

      queues[name] = {
        priority: priority,
        # list is a redis list key for a queue
        list: "#{QUEUE_PREFIX}#{name}",
      }
    end
    
    queues
  end
end

#redis_options ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/jiggler/config.rb', line 140

def redis_options
  @redis_options ||= begin
    opts = @options.slice(
      :concurrency,
      :redis_url
    )

    if at_least_once?
      # for acknowledgers
      opts[:concurrency] *= 2
      # for queue fetchers
      opts[:concurrency] += @options[:fetchers_concurrency] * sorted_queues.count
      # extra poller task to cleanup leftover in-process queues
      opts[:concurrency] += 1
    end

    opts[:concurrency] += 2 # monitor + safety margin
    opts[:concurrency] += 1 if @options[:poller_enabled]
    opts[:async] = true

    opts
  end
end

#redis_pool ⇒ Object 



177
178
179
 
# File 'lib/jiggler/config.rb', line 177

def redis_pool
  @redis_pool ||= Jiggler::RedisStore.new(redis_options).pool
end

#retries_set ⇒ Object 



50
51
52
 
# File 'lib/jiggler/config.rb', line 50

def retries_set
  RETRIES_SET
end

#scheduled_set ⇒ Object 



54
55
56
 
# File 'lib/jiggler/config.rb', line 54

def scheduled_set
  SCHEDULED_SET
end

#server_prefix ⇒ Object

jiggler main process prefix



67
68
69
 
# File 'lib/jiggler/config.rb', line 67

def server_prefix
  SERVER_PREFIX
end

#sorted_lists ⇒ Object 



116
117
118
 
# File 'lib/jiggler/config.rb', line 116

def sorted_lists
  @sorted_lists ||= sorted_queues_data.map { |_, v| v[:list] }
end

#sorted_queues ⇒ Object 



120
121
122
 
# File 'lib/jiggler/config.rb', line 120

def sorted_queues
  @sorted_queues ||= sorted_queues_data.map { |k, _| k }
end

#sorted_queues_data ⇒ Object

sort in descending order (higher priority first)



112
113
114
 
# File 'lib/jiggler/config.rb', line 112

def sorted_queues_data
  @sorted_queues_data ||= queues_data.sort_by { |_, v| -v[:priority] }
end

#summary ⇒ Object 



195
196
197
 
# File 'lib/jiggler/config.rb', line 195

def summary
  @summary ||= Jiggler::Summary.new(self)
end

#with_async_redis ⇒ Object 



124
125
126
127
128
129
130
 
# File 'lib/jiggler/config.rb', line 124

def with_async_redis
  Async do
    redis_pool.acquire do |conn|
      yield conn
    end
  end
end

#with_sync_redis ⇒ Object 



132
133
134
135
136
137
138
 
# File 'lib/jiggler/config.rb', line 132

def with_sync_redis
  Sync do
    redis_pool.acquire do |conn|
      yield conn
    end
  end
end