Class: Lono::FileUploader

Inherits:
Object
  • Object
show all
Extended by:
Memoist
Includes:
Template::AwsService
Defined in:
lib/lono/file_uploader.rb

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ FileUploader

Returns a new instance of FileUploader.



6
7
8
9
10
 
# File 'lib/lono/file_uploader.rb', line 6

def initialize(options={})
  @options = options
  @checksums = {}
  @prefix = "#{folder_key}/#{Lono.env}/files" # s3://s3-bucket/folder/development/files
end

Instance Method Details

#folder_key ⇒ Object

The folder_key is the s3_folder setting with the s3 bucket.

Example:

s3_bucket('s3://mybucket/files/storage/path')
=> files/storage/path



101
102
103
104
105
 
# File 'lib/lono/file_uploader.rb', line 101

def folder_key
  return nil if @options[:noop] # to get spec passing
  return nil unless s3_folder
  s3_folder.sub('s3://','').split('/')[1..-1].join('/')
end

#load_checksums! ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/lono/file_uploader.rb', line 23

def load_checksums!
  resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
  resp.contents.each do |object|
    # key does not include the bucket name
    #    full path = s3://my-bucket/s3_folder/files/production/my-template.yml
    #    key = s3_folder/files/production/my-template.yml
    # etag is the checksum as long as the file is not a multi-part file upload
    # it has extra double quotes wrapped around it.
    #    etag = "\"9cb437490cee2cc96101baf326e5ca81\""
    @checksums[object.key] = strip_surrounding_quotes(object.etag)
  end
  @checksums
end

#md5(path) ⇒ Object

used for file_s3_key helper



38
39
40
 
# File 'lib/lono/file_uploader.rb', line 38

def md5(path)
  Digest::MD5.file(path).to_s[0..7]
end

#md5_key(path) ⇒ Object 



43
44
45
46
47
48
49
50
51
 
# File 'lib/lono/file_uploader.rb', line 43

def md5_key(path)
  pretty_path = path.sub(/^\.\//, '')
  key = "#{@prefix}/#{pretty_path.sub(%r{app/files/},'')}"
  # add the short md5sum to the file
  key = key.sub(/\.(\w+)$/,'') # strip extension
  ext = $1
  md5 = md5(path)
  "#{key}-#{md5}.#{ext}"
end

#remote_checksum(key) ⇒ Object

key example: cloudformation/production/files/lifecycle-0719ab81.zip s3 path: s3://boltops-dev/cloudformation/production/files/lifecycle-0719ab81.zip s3_folder: s3://boltops-dev/cloudformation



83
84
85
 
# File 'lib/lono/file_uploader.rb', line 83

def remote_checksum(key)
  @checksums[key]
end

#s3 ⇒ Object 



115
116
117
 
# File 'lib/lono/file_uploader.rb', line 115

def s3
  @s3 ||= Aws::S3::Client.new
end

#s3_bucket ⇒ Object 



87
88
89
 
# File 'lib/lono/file_uploader.rb', line 87

def s3_bucket
  s3_folder.sub('s3://','').split('/').first
end

#s3_folder ⇒ Object 



91
92
93
94
 
# File 'lib/lono/file_uploader.rb', line 91

def s3_folder
  setting = Lono::Setting.new
  setting.s3_folder
end

#s3_resource ⇒ Object 



111
112
113
 
# File 'lib/lono/file_uploader.rb', line 111

def s3_resource
  @s3_resource ||= Aws::S3::Resource.new
end

#s3_upload(path) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/lono/file_uploader.rb', line 53

def s3_upload(path)
  pretty_path = path.sub(/^\.\//, '')
  key = md5_key(path)
  s3_full_path = "s3://#{s3_bucket}/#{key}"

  local_checksum = Digest::MD5.hexdigest(IO.read(path))
  remote_checksum = remote_checksum(key)
  if local_checksum == remote_checksum
    puts("Not modified: #{pretty_path} to #{s3_full_path}".colorize(:yellow)) unless @options[:noop]
    return # do not upload unless the checksum has changed
  else
    # Example output:
    # Uploaded: app/files/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
    # Uploaded: app/files/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
    message = "Uploading: #{pretty_path} to #{s3_full_path}".colorize(:green)
    message = "NOOP: #{message}" if @options[:noop]
    puts message
  end

  resp = s3.put_object(
    body: IO.read(path),
    bucket: s3_bucket,
    key: key,
    storage_class: "REDUCED_REDUNDANCY"
  ) unless @options[:noop]
end

#strip_surrounding_quotes(string) ⇒ Object 



107
108
109
 
# File 'lib/lono/file_uploader.rb', line 107

def strip_surrounding_quotes(string)
  string.sub(/^"/,'').sub(/"$/,'')
end

#upload_all ⇒ Object 



12
13
14
15
16
17
18
19
20
21
 
# File 'lib/lono/file_uploader.rb', line 12

def upload_all
  puts "Uploading app/files..."
  load_checksums!

  pattern = "#{Lono.root}/app/files/**/*"
  Dir.glob(pattern).each do |path|
    next if ::File.directory?(path)
    s3_upload(path)
  end
end