Class: S3_Multi_Upload::Upload

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_multi_upload.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options) ⇒ Upload

Returns a new instance of Upload.



9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/s3_multi_upload.rb', line 9

def initialize options

  AWS.config :access_key_id     => options[:access_key_id],
             :secret_access_key => options[:secret_access_key]

  @options = options
  @file    = Pathname.new options[:file]

  @s3      = AWS::S3.new
  @bucket  = @s3.buckets.create options[:bucket]
  @object  = @bucket.objects[options[:key] || @file.basename]

  enqueue
end

Instance Attribute Details

#bucket ⇒ Object

Returns the value of attribute bucket.



7
8
9
 
# File 'lib/s3_multi_upload.rb', line 7

def bucket
  @bucket
end

#file ⇒ Object

Returns the value of attribute file.



7
8
9
 
# File 'lib/s3_multi_upload.rb', line 7

def file
  @file
end

#object ⇒ Object

Returns the value of attribute object.



7
8
9
 
# File 'lib/s3_multi_upload.rb', line 7

def object
  @object
end

#options ⇒ Object

Returns the value of attribute options.



7
8
9
 
# File 'lib/s3_multi_upload.rb', line 7

def options
  @options
end

#progress ⇒ Object

Returns the value of attribute progress.



7
8
9
 
# File 'lib/s3_multi_upload.rb', line 7

def progress
  @progress
end

#s3 ⇒ Object

Returns the value of attribute s3.



7
8
9
 
# File 'lib/s3_multi_upload.rb', line 7

def s3
  @s3
end

Instance Method Details

#chunk_size ⇒ Object 



37
38
39
 
# File 'lib/s3_multi_upload.rb', line 37

def chunk_size
  normalize *options[:chunk_size].first
end

#enqueue ⇒ Object 



45
46
47
48
49
 
# File 'lib/s3_multi_upload.rb', line 45

def enqueue
  (file.size.to_f / chunk_size).ceil.times do |index|
    queue << [chunk_size * index, index + 1]
  end
end

#normalize(value, unit = nil) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/s3_multi_upload.rb', line 24

def normalize value, unit = nil
  case unit.downcase.to_sym
  when nil, :b, :byte, :bytes
    value.to_f
  when :k, :kb, :kilobyte, :kilobytes
    value.to_f * 1024
  when :m, :mb, :megabyte, :megabytes
    value.to_f * 1024 ** 2
  when :g, :gb, :gigabyte, :gigabytes
    value.to_f * 1024 ** 3
  end
end

#process ⇒ Object 



78
79
80
81
82
83
84
 
# File 'lib/s3_multi_upload.rb', line 78

def process
  value, unit = *options[:chunk_size].first
  puts "uploading #{file} to s3://#{options[:bucket]}/#{object.key} using #{options[:threads]} threads in chunks of #{value} #{unit}"
  progress if options[:progress_bar]
  abort 'upload failed' unless upload
  progress.finish if options[:progress_bar]
end

#queue ⇒ Object 



41
42
43
 
# File 'lib/s3_multi_upload.rb', line 41

def queue
  @queue ||= Queue.new
end

#upload ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/s3_multi_upload.rb', line 51

def upload
  object.multipart_upload do |upload|
    threads = []

    options[:threads].times do
      threads << Thread.new do
        until queue.empty?
          offset, index = queue.deq :asynchronously rescue nil

          unless offset.nil?
            upload.add_part :data        => file.read(chunk_size, offset),
                            :part_number => index

            progress.inc if options[:progress_bar]
          end
        end
      end
    end

    threads.each &:join
  end
end