Class: S3mirror::Mirror

Inherits:
Object
  • Object
show all
Defined in:
lib/s3mirror.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ Mirror

Returns a new instance of Mirror.



19
20
21
22
23
24
25
26
27
28
 
# File 'lib/s3mirror.rb', line 19

def initialize(options={})
  @temp_destination = Mirror.configuration.temp_download_folder #options.delete(:temp_download_folder)

  if !@temp_destination.blank? && !Pathname.new(@temp_destination).absolute?
    throw 'temp_download_folder should be an absolute path'
  end

  @s3_main = Mirror.configuration.s3_main #options.delete(:s3_main)
  @s3_mirrors = Mirror.configuration.s3_mirrors #options.delete(:s3_mirrors)
end

Class Method Details

.configuration ⇒ Object 



11
12
13
 
# File 'lib/s3mirror.rb', line 11

def self.configuration
  @configuration ||= Configuration.new
end

.configure {|configuration| ... } ⇒ Object

Yields:



15
16
17
 
# File 'lib/s3mirror.rb', line 15

def self.configure(&block)
  yield(configuration)
end

Instance Method Details

#download(key, destination_on_desk) ⇒ Object

read note (1) below



98
99
100
101
102
103
104
105
 
# File 'lib/s3mirror.rb', line 98

def download(key, destination_on_desk)
  # returns true if downloaded successfully
  begin
    s3_object(key).download_file(destination_on_desk)
  rescue Aws::S3::Errors::NoSuchKey
    return false
  end
end

#get_s3_bucket(service) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/s3mirror.rb', line 30

def get_s3_bucket(service)
  # create a client that will be used by resource, this help switching between different s3 providers without changing default configuration set in initializer (i.e: Aws.config.update({...}))
  client = Aws::S3::Client.new(
    endpoint: service[:endpoint],
    region: service[:region],
    credentials: Aws::Credentials.new(service[:access_key_id], service[:secret_access_key])
  )
  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Resource.html
  # if client not supplied, default configuration will be used
  s3 = Aws::S3::Resource.new(client: client)
  return s3.bucket(service[:bucket_name])
end

#mirror(key) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/s3mirror.rb', line 43

def mirror(key)
  unique_prefix = 't' + (Time.now.to_f * 1000000).to_s

  # filename is the name of file used for our download
  filename = File.basename(key)

  # add a temporary unique prefix to the filename to prevent collision (i.e: 2 downloaded file with same name)
  prefixed_filename = "#{unique_prefix}_#{filename}"

  # all files will be downloaded to your app "tmp" folder (otherwise the specified @temp_destination), the unique prefix above will guarantee uniqueness of files names
  if @temp_destination.blank?
    file_path = Rails.root.join('tmp', prefixed_filename)
  else
    file_path = File.join(@temp_destination, prefixed_filename)
  end

  uploaded_now = 0 # uploaded count for the current mirroring try
  uploaded_total = 0

  if download(key, file_path)
    begin
      @s3_mirrors.each do |mirror|
        mirror_bucket = get_s3_bucket(mirror)
        if !mirror_has_file?(key, mirror_bucket)
          uploaded = upload(file_path, key, mirror_bucket)
          if uploaded
            uploaded_now += 1
            uploaded_total += 1
          end
        else
          uploaded_total += 1
        end
      end
    ensure
      # remove downloaded file
      File.delete(file_path)
    end
  else
    # throw 'file not downloaded!!'
  end

  return {
    mirrored_now: uploaded_now,
    total_mirrored: uploaded_total,
    total_failed: @s3_mirrors.size - uploaded_total
  }
end

#mirror_has_file?(file_key, mirror_bucket) ⇒ Boolean

Returns:

  • (Boolean) 


116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/s3mirror.rb', line 116

def mirror_has_file?(file_key, mirror_bucket)
  begin
    object = mirror_bucket.object(file_key)
    if object.exists?
      true
    else
      false
    end
  rescue Aws::S3::Errors::NotFound
    false
  end
end

#s3_object(key) ⇒ Object

get object from main/primary s3 service



92
93
94
95
 
# File 'lib/s3mirror.rb', line 92

def s3_object(key)
  bucket = get_s3_bucket(@s3_main)
  bucket.object(key)
end

#upload(file_path, file_key, mirror_bucket) ⇒ Object

read note (2) below



108
109
110
111
112
113
114
 
# File 'lib/s3mirror.rb', line 108

def upload(file_path, file_key, mirror_bucket)
  object_name = file_key
  # create empty object in the mirror bucket with same name as the original file.
  object = mirror_bucket.object(object_name)
  # upload file to that object.
  object.upload_file(file_path) # return true if uploaded successfully)
end