Class: Range

Inherits:
Object show all
Defined in:
lib/jetpants/monkeypatch.rb

Instance Method Summary collapse

Instance Method Details

#in_chunks(chunks, thread_limit = 40, min_per_chunk = 1) ⇒ Object

Supply a block taking |chunk_min_id, chunk_max_id|. This will execute the block in parallel chunks, supplying the min and max id (inclusive) of the current chunk. Note that thread_limit is capped at the value of Jetpants.max_concurrency.



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/jetpants/monkeypatch.rb', line 78

def in_chunks(chunks, thread_limit=40, min_per_chunk=1)
  per_chunk = ((max - min + 1) / chunks).ceil
  per_chunk = min_per_chunk if per_chunk < min_per_chunk

  min_id_queue = []
  results = []
  min.step(max, per_chunk) {|n| min_id_queue << n}
  min_id_queue.reverse!

  lock = Mutex.new
  group = ThreadGroup.new
  thread_count = Jetpants.max_concurrency
  thread_count = thread_limit if thread_limit && thread_limit < Jetpants.max_concurrency
  thread_count = min_id_queue.length if min_id_queue.length < thread_count
  thread_count.times do
    th = Thread.new do
      while min_id_queue.length > 0 do
        my_min = nil
        lock.synchronize {my_min = min_id_queue.pop}
        break unless my_min
        my_max = my_min + per_chunk - 1
        my_max = max if my_max > max
        result = yield my_min, my_max
        lock.synchronize {results << result}
      end
    end
    group.add(th)
  end
  group.list.each {|th| th.join}
  results
end