Class: Lister

Inherits:
Runnable show all
Defined in:
lib/s3-object-processor.rb

Instance Method Summary collapse

Methods inherited from Runnable

#join, #on_finish

Constructor Details

#initialize(bucket, key_queue, fetch_size, max_keys = nil) ⇒ Lister

Returns a new instance of Lister.



33
34
35
36
37
38
 
# File 'lib/s3-object-processor.rb', line 33

def initialize(bucket, key_queue, fetch_size, max_keys = nil)
	@bucket = bucket
	@key_queue = key_queue
	@fetch_size = fetch_size
	@max_keys = max_keys
end

Instance Method Details

#on_keys_chunk(&callback) ⇒ Object 



40
41
42
43
 
# File 'lib/s3-object-processor.rb', line 40

def on_keys_chunk(&callback)
	@on_keys_chunk = callback
	self
end

#run(prefix = nil) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/s3-object-processor.rb', line 45

def run(prefix = nil)
	super() do
		catch :done do
			marker = ''
			total_keys = 0
			loop do
				keys_chunk = @bucket.keys(prefix: prefix, 'max-keys' => @fetch_size, marker: marker)
				break if keys_chunk.empty?
				@on_keys_chunk.call(keys_chunk) if @on_keys_chunk
				keys_chunk.each do |key|
					throw :done if @max_keys and total_keys >= @max_keys
					@key_queue << key
					total_keys += 1
				end
				marker = keys_chunk.last.name
			end
		end
	end
end