Class: Cnvrg::Downloader::Client
- Inherits:
-
Object
- Object
- Cnvrg::Downloader::Client
- Defined in:
- lib/cnvrg/downloader/client.rb
Direct Known Subclasses
Cnvrg::Downloader::Clients::AzureClient, Cnvrg::Downloader::Clients::GcpClient, Cnvrg::Downloader::Clients::S3Client
Class Method Summary collapse
Instance Method Summary collapse
- #cut_prefix(prefix, file) ⇒ Object
- #decrypt(str) ⇒ Object
- #download(storage_path, local_path) ⇒ Object
- #extract_key_iv(sts_path) ⇒ Object
-
#initialize(params) ⇒ Client
constructor
A new instance of Client.
- #mkdir(path, recursive: false) ⇒ Object
- #prepare_download(local_path) ⇒ Object
- #safe_upload(storage_path, local_path) ⇒ Object
- #upload(storage_path, local_path) ⇒ Object
Constructor Details
#initialize(params) ⇒ Client
Returns a new instance of Client.
9 10 11 12 13 14 |
# File 'lib/cnvrg/downloader/client.rb', line 9 def initialize(params) @key = '' @iv = '' @client = '' @bucket = '' end |
Class Method Details
.factory(params) ⇒ Object
75 76 77 78 79 80 81 82 83 84 85 86 |
# File 'lib/cnvrg/downloader/client.rb', line 75 def self.factory(params) params = params.as_json case params["storage"] when 's3', 'minio' return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"]) when 'azure' azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts]) return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params) when 'gcp' return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"]) end end |
Instance Method Details
#cut_prefix(prefix, file) ⇒ Object
22 23 24 |
# File 'lib/cnvrg/downloader/client.rb', line 22 def cut_prefix(prefix, file) file.gsub(prefix, '').gsub(/^\/*/, '') end |
#decrypt(str) ⇒ Object
42 43 44 |
# File 'lib/cnvrg/downloader/client.rb', line 42 def decrypt(str) Cnvrg::Helpers.decrypt(@key, @iv, str) end |
#download(storage_path, local_path) ⇒ Object
26 27 28 |
# File 'lib/cnvrg/downloader/client.rb', line 26 def download(storage_path, local_path) ### need to be implemented.. end |
#extract_key_iv(sts_path) ⇒ Object
16 17 18 19 20 |
# File 'lib/cnvrg/downloader/client.rb', line 16 def extract_key_iv(sts_path) sts = open(sts_path).read rescue nil raise StandardError.new("Cant open sts") if sts.blank? sts.split("\n") end |
#mkdir(path, recursive: false) ⇒ Object
34 35 36 |
# File 'lib/cnvrg/downloader/client.rb', line 34 def mkdir(path, recursive: false) recursive ? FileUtils.mkdir_p(path) : FileUtils.mkdir(path) end |
#prepare_download(local_path) ⇒ Object
38 39 40 |
# File 'lib/cnvrg/downloader/client.rb', line 38 def prepare_download(local_path) mkdir(File.dirname(local_path), recursive: true) end |
#safe_upload(storage_path, local_path) ⇒ Object
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
# File 'lib/cnvrg/downloader/client.rb', line 46 def safe_upload(storage_path, local_path) n = 1 error = nil while n <= RETRIES begin self.upload(storage_path, local_path) error = nil break rescue => e backoff_time_seconds = backoff_time(n) = "Got error: #{e.class.name} with message: #{e.} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}" if n < RETRIES += ", next retry in: #{backoff_time_seconds} seconds" else += ", done retry, continuing to the next file" end Cnvrg::Logger.() sleep backoff_time_seconds n += 1 error = e end end raise error if error.present? true end |
#upload(storage_path, local_path) ⇒ Object
30 31 32 |
# File 'lib/cnvrg/downloader/client.rb', line 30 def upload(storage_path, local_path) ### need to be implemented.. end |