Class: Fog::Storage::AzureRM::Real

Inherits:
Object
  • Object
show all
Defined in:
lib/fog/azurerm/storage.rb,
lib/fog/azurerm/requests/storage/get_blob.rb,
lib/fog/azurerm/requests/storage/copy_blob.rb,
lib/fog/azurerm/requests/storage/list_blobs.rb,
lib/fog/azurerm/requests/storage/create_disk.rb,
lib/fog/azurerm/requests/storage/delete_blob.rb,
lib/fog/azurerm/requests/storage/delete_disk.rb,
lib/fog/azurerm/requests/storage/get_blob_url.rb,
lib/fog/azurerm/requests/storage/put_blob_block.rb,
lib/fog/azurerm/requests/storage/put_blob_pages.rb,
lib/fog/azurerm/requests/storage/save_page_blob.rb,
lib/fog/azurerm/requests/storage/list_containers.rb,
lib/fog/azurerm/requests/storage/create_container.rb,
lib/fog/azurerm/requests/storage/create_page_blob.rb,
lib/fog/azurerm/requests/storage/delete_container.rb,
lib/fog/azurerm/requests/storage/create_block_blob.rb,
lib/fog/azurerm/requests/storage/get_blob_http_url.rb,
lib/fog/azurerm/requests/storage/get_container_acl.rb,
lib/fog/azurerm/requests/storage/get_container_url.rb,
lib/fog/azurerm/requests/storage/put_blob_metadata.rb,
lib/fog/azurerm/requests/storage/put_container_acl.rb,
lib/fog/azurerm/requests/storage/acquire_blob_lease.rb,
lib/fog/azurerm/requests/storage/commit_blob_blocks.rb,
lib/fog/azurerm/requests/storage/copy_blob_from_uri.rb,
lib/fog/azurerm/requests/storage/get_blob_https_url.rb,
lib/fog/azurerm/requests/storage/release_blob_lease.rb,
lib/fog/azurerm/requests/storage/get_blob_properties.rb,
lib/fog/azurerm/requests/storage/get_storage_account.rb,
lib/fog/azurerm/requests/storage/put_blob_properties.rb,
lib/fog/azurerm/requests/storage/list_storage_accounts.rb,
lib/fog/azurerm/requests/storage/check_container_exists.rb,
lib/fog/azurerm/requests/storage/create_storage_account.rb,
lib/fog/azurerm/requests/storage/delete_storage_account.rb,
lib/fog/azurerm/requests/storage/put_container_metadata.rb,
lib/fog/azurerm/requests/storage/update_storage_account.rb,
lib/fog/azurerm/requests/storage/acquire_container_lease.rb,
lib/fog/azurerm/requests/storage/compare_container_blobs.rb,
lib/fog/azurerm/requests/storage/get_storage_access_keys.rb,
lib/fog/azurerm/requests/storage/release_container_lease.rb,
lib/fog/azurerm/requests/storage/get_container_properties.rb,
lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb,
lib/fog/azurerm/requests/storage/list_storage_account_for_rg.rb,
lib/fog/azurerm/requests/storage/check_storage_account_exists.rb,
lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb,
lib/fog/azurerm/requests/storage/check_storage_account_name_availability.rb

Overview

This class provides the actual implementation for service calls.

Defined Under Namespace

Classes: BlobChunk, BlobFileStream, BlockChunk, BlockFileStream

Constant Summary collapse

BLOCK_SIZE =

32 MB

32 * 1024 * 1024

Instance Method Summary collapse

Constructor Details

#initialize(options) ⇒ Real

Returns a new instance of Real.



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/fog/azurerm/storage.rb', line 90

def initialize(options)
  begin
    require 'azure_mgmt_storage'
    require 'azure/storage'
    require 'securerandom'
    require 'vhd'
    @debug = ENV['DEBUG'] || options[:debug]
    require 'azure/core/http/debug_filter' if @debug
  rescue LoadError => e
    retry if require('rubygems')
    raise e.message
  end

  options[:environment] = 'AzureCloud' if options[:environment].nil?

  @tenant_id = options[:tenant_id]
  @client_id = options[:client_id]
  @client_secret = options[:client_secret]
  @subscription_id = options[:subscription_id]
  @environment = options[:environment]

  credentials = Fog::Credentials::AzureRM.get_credentials(@tenant_id, @client_id, @client_secret, @environment)
  telemetry = "fog-azure-rm/#{Fog::AzureRM::VERSION}"
  unless credentials.nil?
    @storage_mgmt_client = ::Azure::ARM::Storage::StorageManagementClient.new(credentials, resource_manager_endpoint_url(@environment))
    @storage_mgmt_client.subscription_id = @subscription_id
    @storage_mgmt_client.add_user_agent_information(telemetry)
  end

  return unless @azure_storage_account_name != options[:azure_storage_account_name] ||
                @azure_storage_access_key != options[:azure_storage_access_key]

  @azure_storage_account_name = options[:azure_storage_account_name]
  @azure_storage_access_key = options[:azure_storage_access_key]

  azure_client = Azure::Storage::Client.create(storage_account_name: @azure_storage_account_name,
                                               storage_access_key: @azure_storage_access_key)
  azure_client.storage_blob_host = get_blob_endpoint(@azure_storage_account_name, true, @environment)
  @blob_client = azure_client.blob_client
  @blob_client.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
  @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
  @signature_client = Azure::Storage::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
                                                                            @azure_storage_access_key)
end

Instance Method Details

#acquire_blob_lease(container_name, name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/acquire_blob_lease.rb', line 6

def acquire_blob_lease(container_name, name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Leasing blob: #{name} of container #{container_name} options: #{options}"
  Fog::Logger.debug msg

  begin
    lease_id = @blob_client.acquire_blob_lease(container_name, name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Blob #{name} leased successfully."
  lease_id
end

#acquire_container_lease(name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/acquire_container_lease.rb', line 6

def acquire_container_lease(name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Leasing container: #{name} options: #{options}"
  Fog::Logger.debug msg

  begin
    lease_id = @blob_client.acquire_container_lease(name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Container #{name} leased successfully."
  lease_id
end

#check_container_exists(name) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/check_container_exists.rb', line 6

def check_container_exists(name)
  msg = "Checking container #{name}."
  Fog::Logger.debug msg
  begin
    get_container_properties(name)
    Fog::Logger.debug "Container #{name} exists."
    true
  rescue Exception => e
    if e.message.include? 'NotFound'
      Fog::Logger.debug "The specified container #{name} does not exist."
      false
    end
  end
end

#check_storage_account_exists(resource_group_name, storage_account_name) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/fog/azurerm/requests/storage/check_storage_account_exists.rb', line 6

def (resource_group_name, )
  msg = "Checking Storage Account: #{}."
  Fog::Logger.debug msg
  begin
     = @storage_mgmt_client.storage_accounts.get_properties(resource_group_name, )
  rescue MsRestAzure::AzureOperationError => e
    if resource_not_found?(e)
      Fog::Logger.debug "Storage Account #{} doesn't exist."
      return false
    else
      raise_azure_exception(e, msg)
    end
  end

  if .provisioning_state == 'Succeeded'
    Fog::Logger.debug "Storage Account #{} exists."
    true
  else
    Fog::Logger.debug "Storage Account #{} doesn't exist."
    false
  end
end

#check_storage_account_name_availability(storage_account_name, storage_account_type) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/fog/azurerm/requests/storage/check_storage_account_name_availability.rb', line 6

def (, )
  msg = "Checking Storage Account Name availability: #{}."
  Fog::Logger.debug msg

   = (, )
  begin
    name_available_obj = @storage_mgmt_client.storage_accounts.check_name_availability()
  rescue MsRestAzure::AzureOperationError => e
    raise_azure_exception(e, msg)
  end
  if name_available_obj.name_available
    Fog::Logger.debug "Name: #{} is available."
    true
  else
    Fog::Logger.debug "Name: #{} is not available."
    Fog::Logger.debug "Reason: #{name_available_obj.reason}."
    false
  end
end

#commit_blob_blocks(container_name, blob_name, blocks, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/commit_blob_blocks.rb', line 6

def commit_blob_blocks(container_name, blob_name, blocks, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "commit_blob_blocks: Complete uploading #{blob_name} to the container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.commit_blob_blocks(container_name, blob_name, blocks, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Block blob #{blob_name} is uploaded successfully."
  true
end

#compare_container_blobs(container1, container2) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
 
# File 'lib/fog/azurerm/requests/storage/compare_container_blobs.rb', line 6

def compare_container_blobs(container1, container2)
  msg = "Comparing blobs from container #{container1} to container #{container2}"
  Fog::Logger.debug msg
  begin
    identical_blobs = get_identical_blobs_from_containers(container1, container2)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end
  identical_blobs
end

#copy_blob(destination_container, destination_blob, source_container, source_blob, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/copy_blob.rb', line 6

def copy_blob(destination_container, destination_blob, source_container, source_blob, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Copying blob: #{source_blob} from container #{source_container} to container #{destination_container} options: #{options}"
  Fog::Logger.debug msg

  begin
    copy_id, copy_status = @blob_client.copy_blob(destination_container, destination_blob, source_container, source_blob, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Copying blob: x-ms-copy-id: #{copy_id}, x-ms-copy-status: #{copy_status}"
  [copy_id, copy_status]
end

#copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/copy_blob_from_uri.rb', line 6

def copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Copying blob: #{source_blob_uri} to container #{destination_container} options: #{options}"
  Fog::Logger.debug msg

  begin
    copy_id, copy_status = @blob_client.copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Copying blob: x-ms-copy-id: #{copy_id}, x-ms-copy-status: #{copy_status}"
  [copy_id, copy_status]
end

#create_block_blob(container_name, blob_name, body, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/fog/azurerm/requests/storage/create_block_blob.rb', line 6

def create_block_blob(container_name, blob_name, body, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    if body.nil?
      data = nil
    elsif body.respond_to?(:read)
      if body.respond_to?(:rewind)
        begin
          body.rewind
        rescue
          nil
        end
      end
      data = body.read
    else
      data = Fog::Storage.parse_data(body)
      options[:content_length] = data[:headers]['Content-Length']
      options[:content_type] = data[:headers]['Content-Type']
      data = data[:body]
    end

    raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
    blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  if data.nil?
    Fog::Logger.debug "Create a block blob #{blob_name} successfully."
  else
    Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
  end
  blob
end

#create_container(name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/create_container.rb', line 6

def create_container(name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Creating container: #{name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    container = @blob_client.create_container(name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Container #{name} created successfully."
  container
end

#create_disk(disk_name, disk_size_in_gb, options = {}) ⇒ Boolean

Create a disk in Azure storage.

Parameters:

  • disk_name (String) —

    Name of disk

  • disk_size_in_gb (Integer) —

    Disk size in GiB. Value range: [1, 1023]

  • options (Hash) (defaults to: {})

Options Hash (options):

  • container_name (String) —

    Sets name of the container which contains the disk. Default is ‘vhds’.

Returns:

  • (Boolean)

Raises:

  • (ArgumentError) 


15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/fog/azurerm/requests/storage/create_disk.rb', line 15

def create_disk(disk_name, disk_size_in_gb, options = {})
  msg = "Creating disk(#{disk_name}, #{disk_size_in_gb}). options: #{options}"
  Fog::Logger.debug msg

  raise ArgumentError, "disk_size_in_gb #{disk_size_in_gb} must be an integer" unless disk_size_in_gb.is_a?(Integer)
  raise ArgumentError, "Azure minimum disk size is 1 GiB: #{disk_size_in_gb}" if disk_size_in_gb < 1
  raise ArgumentError, "Azure maximum disk size is 1023 GiB: #{disk_size_in_gb}" if disk_size_in_gb > 1023

  container_name = options.delete(:container_name)
  container_name = 'vhds' if container_name.nil?
  blob_name = "#{disk_name}.vhd"
  vhd_size = disk_size_in_gb * 1024 * 1024 * 1024
  blob_size = vhd_size + 512

  opts = {
    type:   :fixed,
    name: '/tmp/footer.vhd', # Only used to initialize vhd, no local file is generated.
    size: disk_size_in_gb
  }
  vhd_footer = Vhd::Library.new(opts).footer.values.join

  begin
    create_page_blob(container_name, blob_name, blob_size, options)
    put_blob_pages(container_name, blob_name, vhd_size, blob_size - 1, vhd_footer, options)
  rescue
    begin
      delete_blob(container_name, blob_name)
    rescue => ex
      Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after create_disk failed. #{ex.inspect}"
    end
    raise
  end

  Fog::Logger.debug "Created a disk #{disk_name} successfully."
  true
end

#create_page_blob(container_name, blob_name, blob_size, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/create_page_blob.rb', line 6

def create_page_blob(container_name, blob_name, blob_size, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "create_page_blob #{blob_name} to the container #{container_name}. blob_size: #{blob_size}, options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.create_page_blob(container_name, blob_name, blob_size, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Page blob #{blob_name} created successfully."
  true
end

#create_storage_account(storage_account_hash) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/fog/azurerm/requests/storage/create_storage_account.rb', line 6

def ()
  msg = "Creating Storage Account: #{[:name]} in Resource Group #{[:resource_group]}."
  Fog::Logger.debug msg
   = ([:sku_name],
                                                      [:location],
                                                      [:replication],
                                                      [:encryption],
                                                      [:tags])
  begin
     = @storage_mgmt_client.storage_accounts.create([:resource_group],
                                                                   [:name],
                                                                   )
  rescue MsRestAzure::AzureOperationError => ex
    raise_azure_exception(ex, msg)
  end
  Fog::Logger.debug 'Storage Account created successfully.'
  
end

#delete_blob(container_name, blob_name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/fog/azurerm/requests/storage/delete_blob.rb', line 6

def delete_blob(container_name, blob_name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Deleting blob: #{blob_name} in container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.delete_blob(container_name, blob_name, options)
  rescue Azure::Core::Http::HTTPError => ex
    return true if ex.message.include?('(404)')
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Blob #{blob_name} deleted successfully."
  true
end

#delete_container(name) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/fog/azurerm/requests/storage/delete_container.rb', line 6

def delete_container(name)
  options = { request_id: SecureRandom.uuid }
  msg = "Deleting container: #{name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.delete_container(name, options)
  rescue Azure::Core::Http::HTTPError => ex
    return true if ex.message.include?('(404)')
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Container #{name} deleted successfully."
  true
end

#delete_disk(disk_name, options = {}) ⇒ Boolean

Delete a disk in Azure storage.

Parameters:

  • disk_name (String) —

    Name of disk

  • options (Hash) (defaults to: {})

Options Hash (options):

  • container_name (String) —

    Sets name of the container which contains the disk. Default is ‘vhds’.

Returns:

  • (Boolean) 


14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/fog/azurerm/requests/storage/delete_disk.rb', line 14

def delete_disk(disk_name, options = {})
  msg = "Deleting disk(#{disk_name}). options: #{options}"
  Fog::Logger.debug msg

  container_name = options.delete(:container_name)
  container_name = 'vhds' if container_name.nil?
  delete_blob(container_name, "#{disk_name}.vhd")

  Fog::Logger.debug "Successfully deleted Disk: #{disk_name}."
  true
end

#delete_storage_account(resource_group, name) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/fog/azurerm/requests/storage/delete_storage_account.rb', line 6

def (resource_group, name)
  msg = "Deleting Storage Account: #{name} in Resource Group #{resource_group}."
  Fog::Logger.debug msg
  begin
    @storage_mgmt_client.storage_accounts.delete(resource_group, name)
  rescue MsRestAzure::AzureOperationError => e
    raise_azure_exception(e, msg)
  end
  Fog::Logger.debug "Storage Account #{name} deleted successfully."
  true
end

#get_blob(container_name, blob_name, options = {}, &block) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/fog/azurerm/requests/storage/get_blob.rb', line 65

def get_blob(container_name, blob_name, options = {}, &block)
  if block_given?
    get_blob_with_block_given(container_name, blob_name, options, &block)
  else
    options[:request_id] = SecureRandom.uuid
    msg = "get_blob blob #{blob_name} in the container #{container_name}. options: #{options}"
    Fog::Logger.debug msg

    begin
      blob, content = @blob_client.get_blob(container_name, blob_name, options)
      Fog::Logger.debug "Get blob #{blob_name} successfully."
      [blob, content]
    rescue Azure::Core::Http::HTTPError => ex
      raise 'NotFound' if ex.message.include?('(404)')
      raise_azure_exception(ex, msg)
    end
  end
end

#get_blob_http_url(container_name, blob_name, expires) ⇒ String

Get an expiring http blob url from Azure blob storage

Parameters:

  • container_name (String) —

    Name of container containing blob

  • blob_name (String) —

    Name of blob to get expiring url for

  • expires (Time) —

    An expiry time for this url

Returns:

  • (String) —

    • http url for blob

See Also:



16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/fog/azurerm/requests/storage/get_blob_http_url.rb', line 16

def get_blob_http_url(container_name, blob_name, expires)
  relative_path = "#{container_name}/#{blob_name}"
  params = {
    service: 'b',
    resource: 'b',
    permissions: 'r',
    expiry: expires.utc.iso8601
  }
  token = @signature_client.generate_service_sas_token(relative_path, params)
  uri = @blob_client.generate_uri(relative_path)
  url = "#{uri}?#{token}"
  url.gsub('https:', 'http:')
end

#get_blob_https_url(container_name, blob_name, expires) ⇒ String

Get an expiring https blob url from Azure blob storage

Parameters:

  • container_name (String) —

    Name of container containing blob

  • blob_name (String) —

    Name of blob to get expiring url for

  • expires (Time) —

    An expiry time for this url

Returns:

  • (String) —

    • https url for blob

See Also:



16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/fog/azurerm/requests/storage/get_blob_https_url.rb', line 16

def get_blob_https_url(container_name, blob_name, expires)
  relative_path = "#{container_name}/#{blob_name}"
  params = {
    service: 'b',
    resource: 'b',
    permissions: 'r',
    expiry: expires.utc.iso8601,
    protocol: 'https'
  }
  token = @signature_client.generate_service_sas_token(relative_path, params)
  uri = @blob_client.generate_uri(relative_path)
  "#{uri}?#{token}"
end

#get_blob_properties(container_name, name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/fog/azurerm/requests/storage/get_blob_properties.rb', line 6

def get_blob_properties(container_name, name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Get Blob #{name} properties in container #{container_name}, options: #{options}."
  Fog::Logger.debug msg
  begin
    blob = @blob_client.get_blob_properties(container_name, name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise 'NotFound' if ex.message.include?('(404)')
    raise_azure_exception(ex, msg)
  end
  Fog::Logger.debug "Getting properties of blob #{name} successfully."
  blob
end

#get_blob_url(container_name, blob_name, options = {}) ⇒ Object

Get a public blob url from Azure blob storage



7
8
9
10
11
12
13
14
15
 
# File 'lib/fog/azurerm/requests/storage/get_blob_url.rb', line 7

def get_blob_url(container_name, blob_name, options = {})
  uri = @blob_client.generate_uri("#{container_name}/#{blob_name}")

  if options[:scheme] == 'http'
    uri.to_s.gsub('https:', 'http:')
  else
    uri.to_s
  end
end

#get_blob_with_block_given(container_name, blob_name, options, &_block) ⇒ Object

Raises:

  • (ArgumentError) 


8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/fog/azurerm/requests/storage/get_blob.rb', line 8

def get_blob_with_block_given(container_name, blob_name, options, &_block)
  options[:request_id] = SecureRandom.uuid
  msg = "get_blob_with_block_given: blob #{blob_name} in the container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    blob = @blob_client.get_blob_properties(container_name, blob_name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise 'NotFound' if ex.message.include?('(404)')
    raise_azure_exception(ex, msg)
  end

  content_length = blob.properties[:content_length]
  if content_length.zero?
    Proc.new.call('', 0, 0)
    return [blob, '']
  end

  start_range = 0
  end_range = content_length - 1
  start_range = options[:start_range] if options[:start_range]
  end_range = options[:end_range] if options[:end_range]
  raise ArgumentError.new(':end_range MUST be greater than :start_range') if start_range > end_range

  if start_range == end_range
    Proc.new.call('', 0, 0)
    return [blob, '']
  end

  buffer_size = BLOCK_SIZE
  buffer_size = options[:block_size] if options[:block_size]
  buffer_start_range = start_range
  total_bytes = end_range - start_range + 1
  params = options.dup

  while buffer_start_range < end_range
    buffer_end_range = [end_range, buffer_start_range + buffer_size - 1].min
    params[:start_range] = buffer_start_range
    params[:end_range] = buffer_end_range
    params[:request_id] = SecureRandom.uuid

    begin
      msg = "get_blob_with_block_given: blob #{blob_name} in the container #{container_name}. options: #{params}"
      Fog::Logger.debug msg
      _, content = @blob_client.get_blob(container_name, blob_name, params)
    rescue Azure::Core::Http::HTTPError => ex
      raise 'NotFound' if ex.message.include?('(404)')
      raise_azure_exception(ex, msg)
    end

    Proc.new.call(content, end_range - buffer_end_range, total_bytes)
    buffer_start_range += buffer_size
  end
  # No need to return content when block is given.
  [blob, '']
end

#get_container_acl(container_name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/get_container_acl.rb', line 6

def get_container_acl(container_name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Get container ACL: #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    container, signed_identifiers = @blob_client.get_container_acl(container_name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Getting ACL of container #{container_name} successfully."
  [container.public_access_level, signed_identifiers]
end

#get_container_properties(name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/fog/azurerm/requests/storage/get_container_properties.rb', line 6

def get_container_properties(name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Getting container properties: #{name}, options: #{options}."
  Fog::Logger.debug msg

  begin
    container = @blob_client.get_container_properties(name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise 'NotFound' if ex.message.include?('(404)')
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Getting properties of container #{name} successfully."
  container
end

#get_container_url(container_name, options = {}) ⇒ String

Get a public container url from Azure storage container

Parameters:

  • container_name (String) —

    Name of container

Returns:

  • (String) —

    • url for container



12
13
14
15
16
17
18
19
20
21
 
# File 'lib/fog/azurerm/requests/storage/get_container_url.rb', line 12

def get_container_url(container_name, options = {})
  query = { 'comp' => 'list', 'restype' => 'container' }
  uri = @blob_client.generate_uri(container_name, query)

  if options[:scheme] == 'http'
    uri.to_s.gsub('https:', 'http:')
  else
    uri.to_s
  end
end

#get_storage_access_keys(resource_group, storage_account_name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/fog/azurerm/requests/storage/get_storage_access_keys.rb', line 6

def get_storage_access_keys(resource_group, , options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Getting storage access keys for storage account: #{}, #{options}."
  Fog::Logger.debug msg
  begin
     = @storage_mgmt_client.storage_accounts.list_keys(resource_group, , options)
  rescue MsRestAzure::AzureOperationError => ex
    raise_azure_exception(ex, msg)
  end
  Fog::Logger.debug "Storage access keys for storage account: #{} listed successfully."
  .keys
end

#get_storage_account(resource_group_name, storage_account_name) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/fog/azurerm/requests/storage/get_storage_account.rb', line 6

def (resource_group_name, )
  msg = "Getting storage account: #{}."
  Fog::Logger.debug msg
  begin
     = @storage_mgmt_client.storage_accounts.get_properties(resource_group_name, )
  rescue MsRestAzure::AzureOperationError => ex
    raise_azure_exception(ex, msg)
  end
  Fog::Logger.debug "Getting storage account: #{} successfully."
  
end

#list_blobs(container_name, options = {}) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/fog/azurerm/requests/storage/list_blobs.rb', line 7

def list_blobs(container_name, options = {})
  options = options.dup
  options[:metadata] = true
  next_marker = nil
  blobs = []

  msg = nil

  max_results = -1
  max_results = options[:max_results].to_i if options[:max_results]
  begin
    loop do
      options[:request_id] = SecureRandom.uuid
      msg = "Listing blobs in container: #{container_name}, options: #{options}"
      Fog::Logger.debug msg
      temp = @blob_client.list_blobs(container_name, options)
      # Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
      raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)

      blobs += temp unless temp.empty?
      break if temp.continuation_token.nil? || temp.continuation_token.empty?
      options[:marker] = temp.continuation_token

      next if max_results == -1

      options[:max_results] = max_results - blobs.size
      if options[:max_results].zero?
        next_marker = temp.continuation_token
        break
      end
    end
  rescue Azure::Core::Http::HTTPError => ex
    raise 'NotFound' if ex.message.include?('(404)')
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Listing blobs in container: #{container_name} successfully."
  {
    next_marker: next_marker,
    blobs: blobs
  }
end

#list_containers ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/fog/azurerm/requests/storage/list_containers.rb', line 7

def list_containers
  options = { metadata: true }
  containers = []
  msg = nil

  begin
    loop do
      options[:request_id] = SecureRandom.uuid
      msg = "Listing containers. options: #{options}"
      Fog::Logger.debug msg
      temp = @blob_client.list_containers(options)
      # Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
      raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)

      containers += temp unless temp.empty?
      break if temp.continuation_token.nil? || temp.continuation_token.empty?
      options[:marker] = temp.continuation_token
    end
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug 'Listing containers successfully.'
  containers
end

#list_storage_account_for_rg(resource_group) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
 
# File 'lib/fog/azurerm/requests/storage/list_storage_account_for_rg.rb', line 6

def (resource_group)
  msg = "Listing Storage Accounts in Resource Group #{resource_group}."
  Fog::Logger.debug msg
  begin
    result = @storage_mgmt_client.storage_accounts.list_by_resource_group(resource_group)
  rescue MsRestAzure::AzureOperationError => ex
    raise_azure_exception(ex, msg)
  end
  result.value
end

#list_storage_accounts ⇒ Object 



6
7
8
9
10
11
12
13
14
15
 
# File 'lib/fog/azurerm/requests/storage/list_storage_accounts.rb', line 6

def list_storage_accounts
  msg = 'Listing Storage Accounts.'
  Fog::Logger.debug msg
  begin
    result = @storage_mgmt_client.storage_accounts.list
  rescue MsRestAzure::AzureOperationError => ex
    raise_azure_exception(ex, msg)
  end
  result.value
end

#multipart_save_block_blob(container_name, blob_name, body, options) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb', line 54

def multipart_save_block_blob(container_name, blob_name, body, options)
  threads_num = options.delete(:worker_thread_num)
  threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1

  begin
    # Initiate the upload
    Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
    content_md5 = options.delete(:content_md5)
    create_block_blob(container_name, blob_name, nil, options)

    # Uploading parts
    Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
    iostream = BlockFileStream.new(body)

    threads = []
    threads_num.times do |id|
      thread = Thread.new do
        Fog::Logger.debug "Created upload thread #{id}."
        while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
          Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
          put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
        end
        Fog::Logger.debug "Upload thread #{id} finished."
      end
      thread.abort_on_exception = true
      threads << thread
    end

    threads.each(&:join)
    # Complete the upload
    options[:content_md5] = content_md5 unless content_md5.nil?
    Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
    commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
  rescue
    # Abort the upload & reraise
    begin
      delete_blob(container_name, blob_name)
    rescue => ex
      Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
    end
    raise
  end

  Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
  true
end

#put_blob_block(container_name, blob_name, block_id, data, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/put_blob_block.rb', line 6

def put_blob_block(container_name, blob_name, block_id, data, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "put_blob_block block_id: #{block_id} / #{blob_name} to the container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.put_blob_block(container_name, blob_name, block_id, data, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "block_id #{block_id} is uploaded successfully."
  true
end

#put_blob_metadata(container_name, name, metadata, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/put_blob_metadata.rb', line 6

def put_blob_metadata(container_name, name, metadata, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Setting Blob #{name} metadata in a container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.set_blob_metadata(container_name, name, metadata, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Setting metadata of blob #{name} successfully."
  true
end

#put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/put_blob_pages.rb', line 6

def put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "put_blob_pages [#{start_range}-#{end_range}] / #{blob_name} to the container #{container_name}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.put_blob_pages(container_name, blob_name, start_range, end_range, data, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "[#{start_range}-#{end_range}] / #{blob_name} is uploaded successfully."
  true
end

#put_blob_properties(container_name, name, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/put_blob_properties.rb', line 6

def put_blob_properties(container_name, name, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Set Blob #{name} properties #{options} in container #{container_name}."
  Fog::Logger.debug msg

  begin
    @blob_client.set_blob_properties(container_name, name, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Setting properties of blob #{name} successfully."
  true
end

#put_container_acl(name, acl, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/put_container_acl.rb', line 6

def put_container_acl(name, acl, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Setting Container #{name} acl #{acl.nil? ? 'nil' : acl}. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.set_container_acl(name, acl, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Setting acl of container #{name} successfully."
  true
end

#put_container_metadata(name, metadata, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/put_container_metadata.rb', line 6

def put_container_metadata(name, metadata, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Setting Container #{name} metadata. options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.set_container_metadata(name, metadata, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Setting metadata of container #{name} successfully."
  true
end

#release_blob_lease(container_name, name, lease_id, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/release_blob_lease.rb', line 6

def release_blob_lease(container_name, name, lease_id, options = {})
  options[:request_id] = SecureRandom.uuid
  msg = "Releasing blob: #{name} of container #{container_name} having lease_id #{lease_id} options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.release_blob_lease(container_name, name, lease_id, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Blob #{name} released successfully."
  true
end

#release_container_lease(name, lease_id, options = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/release_container_lease.rb', line 6

def release_container_lease(name, lease_id, options={})
  options[:request_id] = SecureRandom.uuid
  msg = "Releasing container: #{name} having lease_id #{lease_id} options: #{options}"
  Fog::Logger.debug msg

  begin
    @blob_client.release_container_lease(name, lease_id, options)
  rescue Azure::Core::Http::HTTPError => ex
    raise_azure_exception(ex, msg)
  end

  Fog::Logger.debug "Container #{name} released successfully."
  true
end

#save_page_blob(container_name, blob_name, body, options) ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/fog/azurerm/requests/storage/save_page_blob.rb', line 57

def save_page_blob(container_name, blob_name, body, options)
  threads_num = options.delete(:worker_thread_num)
  threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1

  begin
    blob_size = Fog::Storage.get_body_size(body)
    raise "The page blob size must be aligned to a 512-byte boundary. But the file size is #{blob_size}." if (blob_size % 512).nonzero?

    # Initiate the upload
    Fog::Logger.debug "Creating the page blob #{container_name}/#{blob_name}. options: #{options}"
    create_page_blob(container_name, blob_name, blob_size, options)
    options.delete(:content_md5)

    # Uploading content
    iostream = BlobFileStream.new(body)

    threads = []
    threads_num.times do |id|
      thread = Thread.new do
        Fog::Logger.debug "Created upload thread #{id}."
        while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
          Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, start_range: #{chunk.start_range}, size: #{chunk.data.size}."
          put_blob_pages(container_name, blob_name, chunk.start_range, chunk.end_range, chunk.data, options) if Digest::MD5.hexdigest(chunk.data) != HASH_OF_4MB_EMPTY_CONTENT
        end
        Fog::Logger.debug "Upload thread #{id} finished."
      end
      thread.abort_on_exception = true
      threads << thread
    end

    threads.each(&:join)
  rescue
    # Abort the upload & reraise
    begin
      delete_blob(container_name, blob_name)
    rescue => ex
      Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after save_page_blob failed. #{ex.inspect}"
    end
    raise
  end

  Fog::Logger.debug "Successfully save the page blob: #{container_name}/#{blob_name}."
  true
end

#update_storage_account(storage_account_hash) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/fog/azurerm/requests/storage/update_storage_account.rb', line 6

def ()
  msg = "Updating Storage Account: #{[:name]} in Resource Group #{[:resource_group]}."
  Fog::Logger.debug msg
   = ()
  begin
     = @storage_mgmt_client.storage_accounts.update([:resource_group],
                                                                   [:name],
                                                                   )
  rescue MsRestAzure::AzureOperationError => e
    raise_azure_exception(e, msg)
  end
  Fog::Logger.debug 'Storage Account updated successfully.'
  
end

#wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb', line 6

def wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil)
  begin
    start_time = Time.new
    while copy_status == COPY_STATUS[:PENDING]
      blob = get_blob_properties(container_name, blob_name)
      blob_props = blob.properties
      if !copy_id.nil? && blob_props[:copy_id] != copy_id
        raise "The progress of copying to #{container_name}/#{blob_name} was interrupted by other copy operations."
      end

      copy_status_description = blob_props[:copy_status_description]
      copy_status = blob_props[:copy_status]
      break if copy_status != COPY_STATUS[:PENDING]

      elapse_time = Time.new - start_time
      raise TimeoutError.new("The copy operation cannot be finished in #{timeout} seconds") if !timeout.nil? && elapse_time >= timeout

      copied_bytes, total_bytes = blob_props[:copy_progress].split('/').map(&:to_i)
      interval = copied_bytes.zero? ? 5 : (total_bytes - copied_bytes).to_f / copied_bytes * elapse_time
      interval = 30 if interval > 30
      interval = 1 if interval < 1
      sleep(interval)
    end

    if copy_status != COPY_STATUS[:SUCCESS]
      raise "Failed to copy to #{container_name}/#{blob_name}: \n\tcopy status: #{copy_status}\n\tcopy description: #{copy_status_description}"
    end
  rescue
    # Abort the copy & reraise
    begin
      delete_blob(container_name, blob_name)
    rescue => ex
      Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after the copy operation failed. #{ex.inspect}"
    end
    raise
  end

  Fog::Logger.debug "Successfully copied the blob: #{container_name}/#{blob_name}."
  true
end