Class: Mongo::Dequeue
- Inherits:
-
Object
- Object
- Mongo::Dequeue
- Defined in:
- lib/mongo-dequeue.rb
Overview
heavily inspired by github.com/skiz/mongo_queue
Constant Summary collapse
- DEFAULT_CONFIG =
{ :timeout => 300, :default_priority => 3 }.freeze
Instance Attribute Summary collapse
-
#batch ⇒ Object
readonly
Returns the value of attribute batch.
-
#collection ⇒ Object
readonly
Returns the value of attribute collection.
-
#config ⇒ Object
readonly
Returns the value of attribute config.
Class Method Summary collapse
Instance Method Summary collapse
- #batchprocess ⇒ Object
-
#batchpush(body, item_opts = {}) ⇒ Object
add a new item into the delayed batch.
-
#cleanup ⇒ Object
Removes completed job history.
-
#complete(id) ⇒ Object
Remove the document from the queue.
-
#flush! ⇒ Object
Remove all items from the queue.
-
#initialize(collection, opts = {}) ⇒ Dequeue
constructor
Create a new instance of MongoDequeue with the provided mongodb connection and optional configuration.
- #peek ⇒ Object
-
#pop(opts = {}) ⇒ Object
:id=>“4e039c372b70275e345206e4”.
-
#push(body, item_opts = {}) ⇒ Object
Insert a new item into the queue.
-
#stats ⇒ Object
Provides some information about what is in the queue.
Constructor Details
#initialize(collection, opts = {}) ⇒ Dequeue
Create a new instance of MongoDequeue with the provided mongodb connection and optional configuration. See DEFAULT_CONFIG
for default configuration and possible configuration options.
Example:
db = Mongo::Connection.new('localhost')
config = {:timeout => 90, :attempts => 2}
queue = Mongo::Queue.new(db, config)
23 24 25 26 27 |
# File 'lib/mongo-dequeue.rb', line 23 def initialize(collection, opts={}) @collection = collection @config = DEFAULT_CONFIG.merge(opts) @batch = [] end |
Instance Attribute Details
#batch ⇒ Object (readonly)
Returns the value of attribute batch.
8 9 10 |
# File 'lib/mongo-dequeue.rb', line 8 def batch @batch end |
#collection ⇒ Object (readonly)
Returns the value of attribute collection.
8 9 10 |
# File 'lib/mongo-dequeue.rb', line 8 def collection @collection end |
#config ⇒ Object (readonly)
Returns the value of attribute config.
8 9 10 |
# File 'lib/mongo-dequeue.rb', line 8 def config @config end |
Class Method Details
.generate_duplicate_key(body) ⇒ Object
232 233 234 235 236 237 |
# File 'lib/mongo-dequeue.rb', line 232 def self.generate_duplicate_key(body) return Digest::MD5.hexdigest(body) if body.class == "String" return Digest::MD5.hexdigest(body) if body.class == "Fixnum" #else return Digest::MD5.hexdigest(body.to_json) #won't ever match a duplicate. Need a better way to handle hashes and arrays. end |
Instance Method Details
#batchprocess ⇒ Object
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
# File 'lib/mongo-dequeue.rb', line 72 def batchprocess() js = %Q| function(batch) { var nowutc = new Date(); var ret = []; for(i in batch){ e = batch[i]; //ret.push(e); var query = { 'duplicate_key': e.duplicate_key, 'complete': false, 'locked_at': null }; var object = { '$set': { 'body': e.body, 'inserted_at': nowutc, 'complete': false, 'locked_till': null, 'completed_at': null, 'priority': e.priority, 'duplicate_key': e.duplicate_key, 'completecount': 0 }, '$inc': {'count': 1} }; db.#{collection.name}.update(query, object, true); } return ret; } | cmd = BSON::OrderedHash.new cmd['$eval'] = js cmd['args'] = [@batch] cmd['nolock'] = true result = collection.db.command(cmd) @batch.clear #pp result end |
#batchpush(body, item_opts = {}) ⇒ Object
add a new item into the delayed batch
64 65 66 67 68 69 70 |
# File 'lib/mongo-dequeue.rb', line 64 def batchpush(body, item_opts = {}) @batch << { :body => body, :duplicate_key => item_opts[:duplicate_key] || Mongo::Dequeue.generate_duplicate_key(body), :priority => item_opts[:priority] || @config[:default_priority] } end |
#cleanup ⇒ Object
Removes completed job history
157 158 159 |
# File 'lib/mongo-dequeue.rb', line 157 def cleanup() collection.remove({:complete=>true}); end |
#complete(id) ⇒ Object
Remove the document from the queue. This should be called when the work is done and the document is no longer needed. You must provide the process identifier that the document was locked with to complete it.
142 143 144 145 146 147 148 149 150 151 152 153 154 |
# File 'lib/mongo-dequeue.rb', line 142 def complete(id) begin cmd = BSON::OrderedHash.new cmd['findandmodify'] = collection.name cmd['query'] = {:_id => BSON::ObjectId.from_string(id)} cmd['update'] = {'$set' => {:completed_at => Time.now.utc, :complete => true}, '$inc' => {:completecount => 1} } cmd['limit'] = 1 collection.db.command(cmd) rescue Mongo::OperationFailure => of #opfailure happens when item has been already completed return nil end end |
#flush! ⇒ Object
Remove all items from the queue. Use with caution!
30 31 32 |
# File 'lib/mongo-dequeue.rb', line 30 def flush! collection.drop end |
#peek ⇒ Object
239 240 241 242 243 244 245 246 247 |
# File 'lib/mongo-dequeue.rb', line 239 def peek firstfew = collection.find({ :complete => false, '$or'=>[{:locked_till=> nil},{:locked_till=>{'$lt'=>Time.now.utc}}] }, :sort => [[:priority, :descending],[:inserted_at, :ascending]], :limit => 10) return firstfew end |
#pop(opts = {}) ⇒ Object
:id=>“4e039c372b70275e345206e4”
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
# File 'lib/mongo-dequeue.rb', line 120 def pop(opts = {}) begin timeout = opts[:timeout] || @config[:timeout] cmd = BSON::OrderedHash.new cmd['findandmodify'] = collection.name cmd['update'] = {'$set' => {:locked_till => Time.now.utc+timeout}} cmd['query'] = {:complete => false, '$or'=>[{:locked_till=> nil},{:locked_till=>{'$lt'=>Time.now.utc}}] } cmd['sort'] = {:priority=>-1,:inserted_at=>1} cmd['limit'] = 1 cmd['new'] = true result = collection.db.command(cmd) rescue Mongo::OperationFailure => of return nil end return { :body => result['value']['body'], :id => result['value']['_id'].to_s } end |
#push(body, item_opts = {}) ⇒ Object
Insert a new item into the queue.
Example:
queue.insert(:name => 'Billy', :email => '[email protected]', :message => 'Here is the thing you asked for')
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
# File 'lib/mongo-dequeue.rb', line 38 def push(body, item_opts = {}) dup_key = item_opts[:duplicate_key] || Mongo::Dequeue.generate_duplicate_key(body) selector = { :duplicate_key => dup_key, :complete => false, :locked_at => nil } item = { '$set' => { :body => body, :inserted_at => Time.now.utc, :complete => false, :locked_till => nil, :completed_at => nil, :priority => item_opts[:priority] || @config[:default_priority], :duplicate_key => dup_key, :completecount => 0 }, '$inc' => {:count => 1 } } id = collection.update(selector, item, :upsert => true) end |
#stats ⇒ Object
Provides some information about what is in the queue. We are using an eval to ensure that a lock is obtained during the execution of this query so that the results are not skewed. please be aware that it will lock the database during the execution, so avoid using it too often, even though it it very tiny and should be relatively fast.
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 |
# File 'lib/mongo-dequeue.rb', line 165 def stats js = "function queue_stat(){ return db.eval( function(){ var nowutc = new Date(); var a = db.#{collection.name}.count({'complete': false, '$or':[{'locked_till':null},{'locked_till':{'$lt':nowutc}}] }); var c = db.#{collection.name}.count({'complete': true}); var t = db.#{collection.name}.count(); var l = db.#{collection.name}.count({'complete': false, 'locked_till': {'$gte':nowutc} }); var rc = db.#{collection.name}.group({ 'key': {}, 'cond': {'complete':true}, '$reduce': function(obj, prev){prev.count += (obj.completecount - 1);}, 'initial': {count: 0} }); var p = db.#{collection.name}.group({ 'key': {'priority':1}, 'cond': {}, '$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}}, 'initial': {complete: 0, waiting:0} }); var tasks = db.#{collection.name}.group({ 'key': {'body.task':1}, 'cond': {}, '$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}}, 'initial': {complete: 0, waiting:0} }); return [a, c, t, l, rc[0] ? rc[0].count : 0, p, tasks]; } ); }" #possible additions #db.job_queue.group({ #'key': {'priority':1}, #'cond': {}, #'$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}}, #'initial': {complete: 0, waiting:0} #}); #db.job_queue.group({ #'key': {'body.task':1}, #'cond': {}, #'$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}}, #'initial': {complete: 0, waiting:0} #}); cmd = BSON::OrderedHash.new cmd['$eval'] = js cmd['nolock'] = true available, complete, total, locked, redundant_completes, priority, tasks = collection.db.command(cmd)['retval'] #available, complete, total, locked, redundant_completes, priority, tasks = collection.db.eval(js) { :locked => locked.to_i, :complete => complete.to_i, :available => available.to_i, :total => total.to_i, :redundantcompletes => redundant_completes, :priority => priority, :tasks => tasks } end |