Class: Awscli::S3::Files

Inherits:
Object
  • Object
show all
Defined in:
lib/awscli/s3.rb

Instance Method Summary collapse

Constructor Details

#initialize(connection, options = {}) ⇒ Files

Returns a new instance of Files.



9
10
11
 
# File 'lib/awscli/s3.rb', line 9

def initialize connection, options = {}
  @conn = connection
end

Instance Method Details

#copy_file(source_dir, source_file, dest_dir, dest_file) ⇒ Object 



202
203
204
 
# File 'lib/awscli/s3.rb', line 202

def copy_file(source_dir, source_file, dest_dir, dest_file)
  @conn.directories.get(source_dir).files.get(source_file).copy(dest_dir, dest_file)
end

#delete_file(dir_name, file_name) ⇒ Object 



192
193
194
195
196
197
198
199
200
 
# File 'lib/awscli/s3.rb', line 192

def delete_file(dir_name, file_name)
  #TODO: Handle globs for deletions
  dir = @conn.directories.get(dir_name)
  abort "cannot find bucket: #{dir_name}" unless dir
  remote_file = dir.files.get(file_name)
  abort "cannot find file: #{file_name}" unless remote_file
  remote_file.destroy
  puts "Deleted file: #{file_name}"
end

#download_file(dir_name, file_name, path) ⇒ Object 



179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/awscli/s3.rb', line 179

def download_file(dir_name, file_name, path)
  dir = @conn.directories.get(dir_name)
  abort "cannot find bucket: #{dir_name}" unless dir
  local_path = File.expand_path(path)
  abort "Invalid file path: #{path}" unless File.exist?(local_path)
  remote_file = dir.files.get(file_name)
  abort "cannot find file: #{file_name}" unless remote_file
  File.open("#{local_path}/#{remote_file.key}", 'w') do |f|
    f.write(remote_file.body)
  end
  puts "Downloaded file: #{remote_file.key} to path: #{local_path}"
end

#get_public_url(dir_name, file_name) ⇒ Object 



206
207
208
209
 
# File 'lib/awscli/s3.rb', line 206

def get_public_url(dir_name, file_name)
  url = @conn.directories.get(dir_name).files.get(file_name).public_url
  puts "public url for the file: #{file_name} is #{url}"
end

#list(dir_name, prefix = nil) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/awscli/s3.rb', line 13

def list(dir_name, prefix=nil)
  dir = @conn.directories.get(dir_name)
  abort "cannot find bucket: #{dir_name}" unless dir
  puts "LastModified \t SIZE \t Object"
  if prefix
    dir.files.all(:prefix => prefix).each do |file|
      puts "#{file.last_modified} \t #{file.content_length} \t #{file.key}"
    end
  else
    dir.files.each do |file|
      puts "#{file.last_modified} \t #{file.content_length} \t #{file.key}"
    end
  end
end

#multipart_upload(options) ⇒ Object 



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/awscli/s3.rb', line 105

def multipart_upload(options)
  bucket_name, file_path, tmp_loc, acl = options[:bucket_name], options[:file_path], options[:tmp_dir], options[:acl]
  dir = @conn.directories.get(bucket_name)
  abort "cannot find bucket: #{bucket_name}" unless dir
  file = File.expand_path(file_path)
  abort "Invalid file path: #{file_path}" unless File.exist?(file)
  dest_path = options[:dest_path] if options[:dest_path]
  if dest_path
    if !dest_path.end_with?('/')
      #add trailing slash to detination dir if is not passed
      dest_path = "#{dest_path}/"
    elsif
      #remove leading slash from destination dir path if exists
      dest_path = dest_path[1..-1]
    end
  end

  remote_file = if dest_path
                  "#{dest_path}#{File.basename(file_path)}"
                else
                  "#{File.basename(file_path)}"
                end

  #get the file and split it
  tmp_dir = "#{tmp_loc}/#{File.basename(file_path)}"
  FileUtils.mkdir_p(tmp_dir)
  #split the file into chunks => use smaller chunk sizes to minimize memory
  puts "Spliting the file into 10M chunks ..."
  `split -a3 -b10m #{file} #{tmp_dir}/#{File.basename(file_path)}`
  abort "Cannot perform split on the file" unless $?.to_i == 0

  parts = Dir.glob("#{tmp_loc}/#{File.basename(file_path)}/*").sort

  #initiate the mulitpart upload & store the returned upload_id
  puts "Initializing multipart upload"
  multi_part_up = @conn.initiate_multipart_upload(
    bucket_name,                  # name of the bucket to create object in
    remote_file,                  # name of the object to create
    { 'x-amz-acl' => acl }
  )
  upload_id = multi_part_up.body["UploadId"]
  puts "Upload ID: #{upload_id}"

  part_ids = []

  parts.each_with_index do |part, position|
    part_number = (position + 1).to_s
    puts "Uploading #{part} ..."
    File.open part do |part_file|
      response = @conn.upload_part(
          bucket_name,
          # file[1..-1],
          remote_file,
          upload_id,
          part_number,
          part_file
        )
      part_ids << response.headers['ETag']
    end
  end

  puts "Completing multipart upload ..."
  response = @conn.complete_multipart_upload(
    bucket_name,
    # file[1..-1],
    remote_file,
    upload_id,
    part_ids
  )
  puts "Cleaning tmp_dir ..."
  FileUtils.rm_rf(tmp_dir)
  puts "Successfully completed multipart upload"
end

#upload_file(dir_name, file_path, dest_path = nil) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/awscli/s3.rb', line 28

def upload_file(dir_name, file_path, dest_path=nil)
  dir = @conn.directories.get(dir_name)
  abort "cannot find bucket: #{dir_name}" unless dir
  file = File.expand_path(file_path)
  abort "Invalid file path: #{file_path}" unless File.exist?(file)
  if dest_path && !dest_path.end_with?('/')
    dest_path = "#{dest_path}/"
  end
  file_name = File.basename(file)
  dir.files.create(
      :key => "#{dest_path}#{file_name}",
      :body => File.open(file),
      :public => true
    )
  puts "Uploaded file: #{file_name} to bucket: #{dir_name}"
end

#upload_file_rec(options) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/awscli/s3.rb', line 45

def upload_file_rec(options)
  dir_name, dir_path, threads_count, is_public = options[:bucket_name], options[:dir_path], options[:thread_count], options[:public]
  dest_path = options[:dest_path] if options[:dest_path]
  #check if bucket exists
  bucket = @conn.directories.get(dir_name)
  abort "cannot find bucket: #{dir_name}" unless bucket
  #check if passed path is a dir
  dir = File.expand_path(dir_path)
  abort "dir_path must be a dir" unless File.directory?(dir)
  #add trailing slash to detination dir if is not passed
  if dest_path && !dest_path.end_with?('/')
    dest_path = "#{dest_path}/"
  end
  #remove trailing / from dir_path
  dir = dir.chop if dir.end_with?('/')
  #initializations
  total_size = 0
  files = Queue.new
  threads = Array.new
  semaphore = Mutex.new
  file_number = 0

  Dir.glob("#{dir}/**/*").select { |f| !File.directory?(f) }.each do |file|
    files << file
    total_size += File.size(file)
  end

  total_files = files.size
  puts "Starting Upload using #{threads_count} threads"
  threads_count.times do |count|
    threads << Thread.new do
      # Thread.current[:name] = "upload files #{count}"
      # puts "...started thread '#{Thread.current[:name]}'...\n"
      while not files.empty?
        semaphore.synchronize do
          file_number += 1
        end
        file = files.pop
        key = file.gsub(dir, '')[1..-1]
        dest = "#{dest_path}#{key}"
        puts "[#{file_number}/#{total_files}] Uploading #{key} to s3://#{dir_name}/#{dest}"
        bucket.files.create(
            :key => dest,
            :body => File.open(file),
            :public => is_public
          )
      end
    end
  end
  # Wait for the threads to finish.
  threads.each do |t|
    begin
      t.join
    rescue RuntimeError => e
      puts "Failure on thread #{t[:name]}: #{e.message}"
    end
  end
  puts "Uploaded #{total_files} (#{total_size / 1024} KB)"
end