Class: Falqon::Queue

Inherits:
Object
  • Object
show all
Extended by:
T::Sig
Includes:
Hooks
Defined in:
lib/falqon/queue.rb

Overview

Simple, efficient, and reliable messaging queue implementation

Defined Under Namespace

Classes: Metadata

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Hooks

included, #run_hook

Constructor Details

#initialize(name, retry_strategy: Falqon.configuration.retry_strategy, max_retries: Falqon.configuration.max_retries, retry_delay: Falqon.configuration.retry_delay, redis: Falqon.configuration.redis, logger: Falqon.configuration.logger, version: Falqon::PROTOCOL) ⇒ Queue

Returns a new instance of Queue.



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/falqon/queue.rb', line 71

def initialize(
  name,
  retry_strategy: Falqon.configuration.retry_strategy,
  max_retries: Falqon.configuration.max_retries,
  retry_delay: Falqon.configuration.retry_delay,
  redis: Falqon.configuration.redis,
  logger: Falqon.configuration.logger,
  version: Falqon::PROTOCOL
)
  @name = name
  @id = [Falqon.configuration.prefix, name].compact.join("/")
  @retry_strategy = Strategies.const_get(retry_strategy.to_s.capitalize).new(self)
  @max_retries = max_retries
  @retry_delay = retry_delay
  @redis = redis
  @logger = logger
  @version = version

  redis.with do |r|
    queue_version = r.hget("#{id}:metadata", :version)

    raise Falqon::VersionMismatchError, "Queue #{name} is using protocol version #{queue_version}, but this client is using protocol version #{version}" if queue_version && queue_version.to_i != @version

    r.multi do |t|
      # Register the queue
      t.sadd([Falqon.configuration.prefix, "queues"].compact.join(":"), name)

      # Set creation and update timestamp (if not set)
      t.hsetnx("#{id}:metadata", :created_at, Time.now.to_i)
      t.hsetnx("#{id}:metadata", :updated_at, Time.now.to_i)

      # Set protocol version
      t.hsetnx("#{id}:metadata", :version, @version)
    end
  end

  run_hook :initialize, :after
end

Instance Attribute Details

#id ⇒ Object (readonly)

Returns the value of attribute id.



19
20
21
 
# File 'lib/falqon/queue.rb', line 19

def id
  @id
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



39
40
41
 
# File 'lib/falqon/queue.rb', line 39

def logger
  @logger
end

#max_retries ⇒ Object (readonly)

Returns the value of attribute max_retries.



27
28
29
 
# File 'lib/falqon/queue.rb', line 27

def max_retries
  @max_retries
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



15
16
17
 
# File 'lib/falqon/queue.rb', line 15

def name
  @name
end

#redis ⇒ Object (readonly)

Returns the value of attribute redis.



35
36
37
 
# File 'lib/falqon/queue.rb', line 35

def redis
  @redis
end

#retry_delay ⇒ Object (readonly)

Returns the value of attribute retry_delay.



31
32
33
 
# File 'lib/falqon/queue.rb', line 31

def retry_delay
  @retry_delay
end

#retry_strategy ⇒ Object (readonly)

Returns the value of attribute retry_strategy.



23
24
25
 
# File 'lib/falqon/queue.rb', line 23

def retry_strategy
  @retry_strategy
end

Class Method Details

.all ⇒ Object 



585
586
587
588
589
590
591
 
# File 'lib/falqon/queue.rb', line 585

def all
  Falqon.configuration.redis.with do |r|
    r
      .smembers([Falqon.configuration.prefix, "queues"].compact.join(":"))
      .map { |id| new(id) }
  end
end

.size ⇒ Object 



598
599
600
601
602
603
 
# File 'lib/falqon/queue.rb', line 598

def size
  Falqon.configuration.redis.with do |r|
    r
      .scard([Falqon.configuration.prefix, "queues"].compact.join(":"))
  end
end

Instance Method Details

#clear ⇒ Object 



319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'lib/falqon/queue.rb', line 319

def clear
  logger.debug "Clearing queue #{name}"

  run_hook :clear, :before

  # Clear all sub-queues
  message_ids = pending.clear + processing.clear + scheduled.clear + dead.clear

  redis.with do |r|
    r.multi do |t|
      # Clear metadata
      t.hdel("#{id}:metadata", :processed, :failed, :retried)

      # Set update timestamp
      t.hset("#{id}:metadata", :updated_at, Time.now.to_i)
    end
  end

  run_hook :clear, :after

  # Return identifiers
  message_ids
end

#dead ⇒ Object 



568
569
570
 
# File 'lib/falqon/queue.rb', line 568

def dead
  @dead ||= SubQueue.new(self, "dead")
end

#delete ⇒ Object 



353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
 
# File 'lib/falqon/queue.rb', line 353

def delete
  logger.debug "Deleting queue #{name}"

  run_hook :delete, :before

  # Delete all sub-queues
  [pending, processing, scheduled, dead]
    .each(&:clear)

  redis.with do |r|
    r.multi do |t|
      # Delete metadata
      t.del("#{id}:metadata")

      # Deregister the queue
      t.srem([Falqon.configuration.prefix, "queues"].compact.join(":"), name)
    end
  end

  run_hook :delete, :after
end

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


515
516
517
 
# File 'lib/falqon/queue.rb', line 515

def empty?
  size.zero?
end

#inspect ⇒ Object 



574
575
576
 
# File 'lib/falqon/queue.rb', line 574

def inspect
  "#<#{self.class} name=#{name.inspect} pending=#{pending.size} processing=#{processing.size} scheduled=#{scheduled.size} dead=#{dead.size}>"
end

#metadata ⇒ Object 



525
526
527
528
529
530
 
# File 'lib/falqon/queue.rb', line 525

def metadata
  redis.with do |r|
    Metadata
      .parse(r.hgetall("#{id}:metadata"))
  end
end

#peek(index: 0) ⇒ Object 



257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
 
# File 'lib/falqon/queue.rb', line 257

def peek(index: 0)
  logger.debug "Peeking at next message in queue #{name}"

  run_hook :peek, :before

  # Get identifier from pending queue
  message_id = pending.peek(index:)

  return unless message_id

  run_hook :peek, :after

  # Retrieve data
  Message.new(self, id: message_id).data
end

#pending ⇒ Object 



538
539
540
 
# File 'lib/falqon/queue.rb', line 538

def pending
  @pending ||= SubQueue.new(self)
end

#pop(&block) ⇒ Object 



181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/falqon/queue.rb', line 181

def pop(&block)
  logger.debug "Popping message from queue #{name}"

  run_hook :pop, :before

  message = redis.with do |r|
    # Move identifier from pending queue to processing queue
    message_id = r.blmove(pending.id, processing.id, :left, :right).to_i

    # Get retry count
    retries = r.hget("#{id}:metadata:#{message_id}", :retries).to_i

    r.multi do |t|
      # Set message status
      t.hset("#{id}:metadata:#{message_id}", :status, "processing")

      # Set update timestamp
      t.hset("#{id}:metadata", :updated_at, Time.now.to_i)
      t.hset("#{id}:metadata:#{message_id}", :updated_at, Time.now.to_i)

      # Increment processing counter
      t.hincrby("#{id}:metadata", :processed, 1)

      # Increment retry counter if message is retried
      t.hincrby("#{id}:metadata", :retried, 1) if retries.positive?
    end

    Message.new(self, id: message_id)
  end

  data = message.data

  yield data if block

  run_hook :pop, :after

  # Remove identifier from processing queue
  processing.remove(message.id)

  # Delete message
  message.delete

  data
rescue Error => e
  logger.debug "Error processing message #{message.id}: #{e.message}"

  # Increment failure counter
  redis.with { |r| r.hincrby("#{id}:metadata", :failed, 1) }

  # Retry message according to configured strategy
  retry_strategy.retry(message, e)

  nil
end

#processing ⇒ Object 



548
549
550
 
# File 'lib/falqon/queue.rb', line 548

def processing
  @processing ||= SubQueue.new(self, "processing")
end

#push(*data) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/falqon/queue.rb', line 124

def push(*data)
  logger.debug "Pushing #{data.size} messages onto queue #{name}"

  run_hook :push, :before

  # Set update timestamp
  redis.with { |r| r.hset("#{id}:metadata", :updated_at, Time.now.to_i) }

  ids = data.map do |d|
    message = Message
      .new(self, data: d)
      .create

    # Push identifier to queue
    pending.add(message.id)

    # Set message status
    redis.with { |r| r.hset("#{id}:metadata:#{message.id}", :status, "pending") }

    # Return identifier(s)
    data.size == 1 ? (return message.id) : (next message.id)
  end

  run_hook :push, :after

  # Return identifier(s)
  ids
end

#range(start: 0, stop: -1)) ⇒ Object 



290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/falqon/queue.rb', line 290

def range(start: 0, stop: -1)
  logger.debug "Peeking at next messages in queue #{name}"

  run_hook :range, :before

  # Get identifiers from pending queue
  message_ids = pending.range(start:, stop:)

  return [] unless message_ids.any?

  run_hook :range, :after

  # Retrieve data
  message_ids.map { |id| Message.new(self, id:).data }
end

#refill ⇒ Object 



390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
 
# File 'lib/falqon/queue.rb', line 390

def refill
  logger.debug "Refilling queue #{name}"

  run_hook :refill, :before

  message_ids = []

  # Move all identifiers from tail of processing queue to head of pending queue
  redis.with do |r|
    while (message_id = r.lmove(processing.id, id, :right, :left))
      # Set message status
      r.hset("#{id}:metadata:#{message_id}", :status, "pending")

      message_ids << message_id
    end
  end

  run_hook :refill, :after

  message_ids
end

#revive ⇒ Object 



426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
 
# File 'lib/falqon/queue.rb', line 426

def revive
  logger.debug "Reviving queue #{name}"

  run_hook :revive, :before

  message_ids = []

  # Move all identifiers from tail of dead queue to head of pending queue
  redis.with do |r|
    while (message_id = r.lmove(dead.id, id, :right, :left))
      # Set message status
      r.hset("#{id}:metadata:#{message_id}", :status, "pending")

      message_ids << message_id
    end
  end

  run_hook :revive, :after

  message_ids
end

#schedule ⇒ Object 



464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
 
# File 'lib/falqon/queue.rb', line 464

def schedule
  logger.debug "Scheduling failed messages on queue #{name}"

  run_hook :schedule, :before

  message_ids = T.let([], T::Array[Identifier])

  # Move all due identifiers from scheduled queue to head of pending queue
  redis.with do |r|
    # Select all identifiers that are due (score <= current timestamp)
    # FIXME: replace with zrange(by_score: true) when https://github.com/sds/mock_redis/issues/307 is resolved
    # TODO: work in batches
    message_ids = r.zrangebyscore(scheduled.id, 0, Time.now.to_i).map(&:to_i)

    logger.debug "Scheduling messages #{message_ids.join(', ')} on queue #{name}"

    r.multi do |t|
      message_ids.each do |message_id|
        # Set message status
        t.hset("#{id}:metadata:#{message_id}", :status, "pending")

        # Add identifier to pending queue
        pending.add(message_id)

        # Remove identifier from scheduled queue
        scheduled.remove(message_id)
      end
    end
  end

  run_hook :schedule, :after

  message_ids
end

#scheduled ⇒ Object 



558
559
560
 
# File 'lib/falqon/queue.rb', line 558

def scheduled
  @scheduled ||= SubSet.new(self, "scheduled")
end

#size ⇒ Object 



504
505
506
 
# File 'lib/falqon/queue.rb', line 504

def size
  pending.size
end