Class: Cnvrg::Downloader::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/cnvrg/downloader/client.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params) ⇒ Client

Returns a new instance of Client.



9
10
11
12
13
14
 
# File 'lib/cnvrg/downloader/client.rb', line 9

def initialize(params)
  @key = ''
  @iv = ''
  @client = ''
  @bucket = ''
end

Class Method Details

.factory(params) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/cnvrg/downloader/client.rb', line 75

def self.factory(params)
  params = params.as_json
  case params["storage"]
  when 's3', 'minio'
    return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
  when 'azure'
    azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
    return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
  when 'gcp'
    return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
  end
end

Instance Method Details

#cut_prefix(prefix, file) ⇒ Object 



22
23
24
 
# File 'lib/cnvrg/downloader/client.rb', line 22

def cut_prefix(prefix, file)
  file.gsub(prefix, '').gsub(/^\/*/, '')
end

#decrypt(str) ⇒ Object 



42
43
44
 
# File 'lib/cnvrg/downloader/client.rb', line 42

def decrypt(str)
  Cnvrg::Helpers.decrypt(@key, @iv, str)
end

#download(storage_path, local_path) ⇒ Object 



26
27
28
 
# File 'lib/cnvrg/downloader/client.rb', line 26

def download(storage_path, local_path)
  ### need to be implemented..
end

#extract_key_iv(sts_path) ⇒ Object

Raises:

  • (StandardError) 


16
17
18
19
20
 
# File 'lib/cnvrg/downloader/client.rb', line 16

def extract_key_iv(sts_path)
  sts = open(sts_path).read rescue nil
  raise StandardError.new("Cant open sts") if sts.blank?
  sts.split("\n")
end

#mkdir(path, recursive: false) ⇒ Object 



34
35
36
 
# File 'lib/cnvrg/downloader/client.rb', line 34

def mkdir(path, recursive: false)
  recursive ? FileUtils.mkdir_p(path) : FileUtils.mkdir(path)
end

#prepare_download(local_path) ⇒ Object 



38
39
40
 
# File 'lib/cnvrg/downloader/client.rb', line 38

def prepare_download(local_path)
  mkdir(File.dirname(local_path), recursive: true)
end

#safe_upload(storage_path, local_path) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/cnvrg/downloader/client.rb', line 46

def safe_upload(storage_path, local_path)
  n = 1
  error = nil
  while n <= RETRIES
    begin
      self.upload(storage_path, local_path)
      error = nil
      break
    rescue => e
      backoff_time_seconds = backoff_time(n)

      message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
      if n < RETRIES
        message += ", next retry in: #{backoff_time_seconds} seconds"
      else
        message += ", done retry, continuing to the next file"
      end
      Cnvrg::Logger.log_error_message(message)

      sleep backoff_time_seconds

      n += 1
      error = e
    end
  end
  raise error if error.present?
  true
end

#upload(storage_path, local_path) ⇒ Object 



30
31
32
 
# File 'lib/cnvrg/downloader/client.rb', line 30

def upload(storage_path, local_path)
  ### need to be implemented..
end