Class: Pupistry::StorageAWS
- Inherits:
-
Object
- Object
- Pupistry::StorageAWS
- Defined in:
- lib/pupistry/storage_aws.rb
Overview
Pupistry::StorageAWS
Instance Attribute Summary collapse
-
#bucket ⇒ Object
Returns the value of attribute bucket.
-
#s3 ⇒ Object
Returns the value of attribute s3.
Instance Method Summary collapse
- #download(src, dest = 'stream') ⇒ Object
-
#initialize(mode) ⇒ StorageAWS
constructor
A new instance of StorageAWS.
- #upload(src, dest) ⇒ Object
Constructor Details
#initialize(mode) ⇒ StorageAWS
Returns a new instance of StorageAWS.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
# File 'lib/pupistry/storage_aws.rb', line 13 def initialize(mode) # mode is either "build" or "agent", depending which we load a different # set of permissions. Awareness of both is intentional, since we want the # build machines to known the agent creds so we can generate bootstrap # template files. unless defined? $config['general']['s3_bucket'] $logger.fatal 'You must set the AWS s3_bucket' exit 0 end # Define AWS configuration if defined? $config[mode]['access_key_id'] if $config[mode]['access_key_id'] == '' $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery' $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.' else $logger.debug 'Loading AWS credentials from configuration file' AWS.config( access_key_id: $config[mode]['access_key_id'], secret_access_key: $config[mode]['secret_access_key'], region: $config[mode]['region'], proxy_uri: $config[mode]['proxy_uri'] ) end else $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery' $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.' end # Setup S3 bucket if defined? $config['general']['s3_endpoint'] and $config['general']['s3_endpoint'] != nil $logger.debug 'Connecting to alternative endpoint ' + $config['general']['s3_endpoint'] @s3 = AWS::S3.new( s3_endpoint: $config['general']['s3_endpoint'], s3_force_path_style: true, ) else @s3 = AWS::S3.new end @bucket = @s3.buckets[$config[mode]['s3_bucket']] end |
Instance Attribute Details
#bucket ⇒ Object
Returns the value of attribute bucket.
11 12 13 |
# File 'lib/pupistry/storage_aws.rb', line 11 def bucket @bucket end |
#s3 ⇒ Object
Returns the value of attribute s3.
10 11 12 |
# File 'lib/pupistry/storage_aws.rb', line 10 def s3 @s3 end |
Instance Method Details
#download(src, dest = 'stream') ⇒ Object
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 |
# File 'lib/pupistry/storage_aws.rb', line 96 def download(src, dest = 'stream') $logger.debug "Downloading file s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{src} to #{dest}" begin # Generate the object name/key based on the relative file name and path. s3_obj_name = "#{$config['general']['s3_prefix']}#{src}" s3_obj = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name] # Download the file if dest == 'stream' # Return the contents rather than writing to disk. We assume stream mode # if the dest filename was unspecified return s3_obj.read else # Download to an ondisk file File.open(dest, 'wb') do |file| s3_obj.read do |chunk| file.write(chunk) end end end rescue AWS::S3::Errors::NoSuchKey $logger.debug 'No such file exists for download, this is normal at times.' return false rescue AWS::S3::Errors::NoSuchBucket $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist" exit 0 rescue AWS::S3::Errors::AccessDenied $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied" exit 0 rescue AWS::S3::Errors::InvalidObjectState $logger.warn "Unable to download \"#{src}\", it has been archived off into Glacier and would need to be recovered first." # Do we need to restore it? begin if s3_obj.restore_in_progress? $logger.warn "A restore of this file is currently in progress, but can take up to 4 hours - please re-try later." else # Not being restored currently, let's file a request. This allows # us to cater for situations where a bunch of servers need to get # an old manifest/file, however the fastest solution is to simply # do a new `pupistry push` from a workstation to upload a new # manifest file. if s3_obj.restore(:days => 30) $logger.warn "Recover request has been issued, this could take up to 4 hours to complete." $logger.warn "Note that doing a `pupistry push` from the workstation would solve this faster." end end rescue StandardError => e $logger.error "Glacier restore request for #{src} failed. (#{e.class}), best option is to push the latest manifest from a workstation with `pupistry push`." end return false rescue AWS::S3::Errors::PermanentRedirect => e $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}." raise e rescue AWS::S3::Errors::SignatureDoesNotMatch => e $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials" raise e rescue AWS::S3::Errors::MissingCredentialsError $logger.error 'AWS credentials not supplied. You must either:' $logger.error 'a) Specify them in the config file for Pupistry' $logger.error 'b) Use IAM roles with an EC2 instance.' $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY' return false rescue StandardError => e raise e end end |
#upload(src, dest) ⇒ Object
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
# File 'lib/pupistry/storage_aws.rb', line 57 def upload(src, dest) $logger.debug "Pushing file #{src} to s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{dest}" begin # Generate the object name/key based on the relative file name and path. s3_obj_name = "#{$config['general']['s3_prefix']}#{dest}" s3_obj = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name] # Perform S3 upload s3_obj.write(file: src) rescue AWS::S3::Errors::NoSuchBucket $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist" exit 0 rescue AWS::S3::Errors::AccessDenied $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied" exit 0 rescue AWS::S3::Errors::PermanentRedirect => e $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}." raise e rescue AWS::S3::Errors::SignatureDoesNotMatch => e $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials" raise e rescue AWS::S3::Errors::MissingCredentialsError $logger.error 'AWS credentials not supplied. You must either:' $logger.error 'a) Specify them in the config file for Pupistry' $logger.error 'b) Use IAM roles with an EC2 instance.' $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY' return false rescue StandardError => e raise e end end |