Class: Backup::ChunkingS3Actor

Inherits:
S3Actor
  • Object
show all
Defined in:
lib/backup/s3_helpers.rb

Constant Summary collapse

DEFAULT_MAX_OBJECT_SIZE =

5 * 2^30 = 5GB

5368709120
DEFAULT_CHUNK_SIZE =

4 * 2^30 = 4GB

4294967296

Instance Attribute Summary

Attributes inherited from S3Actor

#config, #rotation

Instance Method Summary collapse

Methods inherited from S3Actor

#cleanup, #verify_rotation_hierarchy_exists

Constructor Details

#initialize(config) ⇒ ChunkingS3Actor

Returns a new instance of ChunkingS3Actor.



94
95
96
97
98
 
# File 'lib/backup/s3_helpers.rb', line 94

def initialize(config)
  super
  @max_object_size = c[:max_object_size] ||= DEFAULT_MAX_OBJECT_SIZE
  @chunk_size = c[:chunk_size] ||= DEFAULT_CHUNK_SIZE
end

Instance Method Details

#delete(object_key) ⇒ Object

Remove a file from s3



126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/backup/s3_helpers.rb', line 126

def delete(object_key)
  puts "delete: #{object_key}"
  # determine if there are multiple objects with this key prefix
  chunks = @bucket.keys(:prefix => object_key)
  if chunks.size > 1
    # delete them all
    chunks.each do |chunk|
      puts "  #{chunk.name}"
      chunk.delete
    end
  else
    chunks.first.delete
  end
end

#put(last_result) ⇒ Object

Send a file to s3



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/backup/s3_helpers.rb', line 101

def put(last_result)
  object_key = Rotator.timestamped_prefix(last_result)
  puts "put: #{object_key}"
  # determine if the file is too large
  if File.stat(last_result).size > @max_object_size
    # if so, split
    split_command = "cd #{File.dirname(last_result)} && split -d -b #{@chunk_size} #{File.basename(last_result)} #{File.basename(last_result)}."
    puts "split: #{split_command}"
    system split_command
    chunks = Dir.glob("#{last_result}.*")
    # put each file in the split
    chunks.each do |chunk|
      chunk_index = chunk.sub(last_result,"")
      chunk_key = "#{object_key}#{chunk_index}"
      puts "  #{chunk_key}"
      @bucket.put(chunk_key, open(chunk))
      FileUtils.rm(chunk)
    end
  else
    @bucket.put(object_key, open(last_result))
  end
  object_key
end