Class: ObjectStorage::DirectUpload
- Inherits:
-
Object
- Object
- ObjectStorage::DirectUpload
- Includes:
- Gitlab::Utils::StrongMemoize
- Defined in:
- lib/object_storage/direct_upload.rb
Overview
The DirectUpload class generates a set of presigned URLs that can be used to upload data to object storage from untrusted component: Workhorse, Runner?
For Google it assumes that the platform supports variable Content-Length.
For AWS it initiates Multipart Upload and presignes a set of part uploads.
Class calculates the best part size to be able to upload up to asked maximum size.
The number of generated parts will never go above 100,
but we will always try to reduce amount of generated parts.
The part size is rounded-up to 5MB.
Constant Summary collapse
- TIMEOUT =
4.hours
- EXPIRE_OFFSET =
15.minutes
- MAXIMUM_MULTIPART_PARTS =
100
- MINIMUM_MULTIPART_SIZE =
5.megabytes
Instance Attribute Summary collapse
-
#bucket_name ⇒ Object
readonly
Returns the value of attribute bucket_name.
-
#config ⇒ Object
readonly
Returns the value of attribute config.
-
#credentials ⇒ Object
readonly
Returns the value of attribute credentials.
-
#has_length ⇒ Object
readonly
Returns the value of attribute has_length.
-
#maximum_size ⇒ Object
readonly
Returns the value of attribute maximum_size.
-
#object_name ⇒ Object
readonly
Returns the value of attribute object_name.
-
#skip_delete ⇒ Object
readonly
Returns the value of attribute skip_delete.
Instance Method Summary collapse
- #azure_gocloud_url ⇒ Object
- #delete_url ⇒ Object
- #get_url ⇒ Object
- #google_gocloud_url ⇒ Object
-
#initialize(config, object_name, has_length:, maximum_size: nil, skip_delete: false) ⇒ DirectUpload
constructor
A new instance of DirectUpload.
- #multipart_abort_url ⇒ Object
- #multipart_complete_url ⇒ Object
- #multipart_part_upload_url(part_number) ⇒ Object
- #multipart_part_urls ⇒ Object
- #multipart_upload_hash ⇒ Object
- #provider ⇒ Object
- #store_url ⇒ Object
- #to_hash ⇒ Object
- #use_workhorse_google_client? ⇒ Boolean
- #use_workhorse_s3_client? ⇒ Boolean
- #workhorse_aws_hash ⇒ Object
- #workhorse_azure_hash ⇒ Object
- #workhorse_client_hash ⇒ Object
- #workhorse_google_hash ⇒ Object
Constructor Details
#initialize(config, object_name, has_length:, maximum_size: nil, skip_delete: false) ⇒ DirectUpload
Returns a new instance of DirectUpload.
28 29 30 31 32 33 34 35 36 37 38 39 40 |
# File 'lib/object_storage/direct_upload.rb', line 28 def initialize(config, object_name, has_length:, maximum_size: nil, skip_delete: false) unless has_length raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size end @config = config @credentials = config.credentials @bucket_name = config.bucket @object_name = object_name @has_length = has_length @maximum_size = maximum_size @skip_delete = skip_delete end |
Instance Attribute Details
#bucket_name ⇒ Object (readonly)
Returns the value of attribute bucket_name.
25 26 27 |
# File 'lib/object_storage/direct_upload.rb', line 25 def bucket_name @bucket_name end |
#config ⇒ Object (readonly)
Returns the value of attribute config.
25 26 27 |
# File 'lib/object_storage/direct_upload.rb', line 25 def config @config end |
#credentials ⇒ Object (readonly)
Returns the value of attribute credentials.
25 26 27 |
# File 'lib/object_storage/direct_upload.rb', line 25 def credentials @credentials end |
#has_length ⇒ Object (readonly)
Returns the value of attribute has_length.
26 27 28 |
# File 'lib/object_storage/direct_upload.rb', line 26 def has_length @has_length end |
#maximum_size ⇒ Object (readonly)
Returns the value of attribute maximum_size.
26 27 28 |
# File 'lib/object_storage/direct_upload.rb', line 26 def maximum_size @maximum_size end |
#object_name ⇒ Object (readonly)
Returns the value of attribute object_name.
25 26 27 |
# File 'lib/object_storage/direct_upload.rb', line 25 def object_name @object_name end |
#skip_delete ⇒ Object (readonly)
Returns the value of attribute skip_delete.
26 27 28 |
# File 'lib/object_storage/direct_upload.rb', line 26 def skip_delete @skip_delete end |
Instance Method Details
#azure_gocloud_url ⇒ Object
112 113 114 115 116 |
# File 'lib/object_storage/direct_upload.rb', line 112 def azure_gocloud_url url = "azblob://#{bucket_name}" url += "?domain=#{config.azure_storage_domain}" if config.azure_storage_domain.present? url end |
#delete_url ⇒ Object
166 167 168 |
# File 'lib/object_storage/direct_upload.rb', line 166 def delete_url connection.delete_object_url(bucket_name, object_name, expire_at) end |
#get_url ⇒ Object
157 158 159 160 161 162 163 |
# File 'lib/object_storage/direct_upload.rb', line 157 def get_url if config.google? connection.get_object_https_url(bucket_name, object_name, expire_at) else connection.get_object_url(bucket_name, object_name, expire_at) end end |
#google_gocloud_url ⇒ Object
131 132 133 |
# File 'lib/object_storage/direct_upload.rb', line 131 def google_gocloud_url "gs://#{bucket_name}" end |
#multipart_abort_url ⇒ Object
204 205 206 207 208 209 210 211 |
# File 'lib/object_storage/direct_upload.rb', line 204 def multipart_abort_url connection.signed_url({ method: 'DELETE', bucket_name: bucket_name, object_name: object_name, query: { 'uploadId' => upload_id } }, expire_at) end |
#multipart_complete_url ⇒ Object
193 194 195 196 197 198 199 200 201 |
# File 'lib/object_storage/direct_upload.rb', line 193 def multipart_complete_url connection.signed_url({ method: 'POST', bucket_name: bucket_name, object_name: object_name, query: { 'uploadId' => upload_id }, headers: { 'Content-Type' => 'application/xml' } }, expire_at) end |
#multipart_part_upload_url(part_number) ⇒ Object
182 183 184 185 186 187 188 189 190 |
# File 'lib/object_storage/direct_upload.rb', line 182 def multipart_part_upload_url(part_number) connection.signed_url({ method: 'PUT', bucket_name: bucket_name, object_name: object_name, query: { 'uploadId' => upload_id, 'partNumber' => part_number }, headers: }, expire_at) end |
#multipart_part_urls ⇒ Object
175 176 177 178 179 |
# File 'lib/object_storage/direct_upload.rb', line 175 def multipart_part_urls Array.new(number_of_multipart_parts) do |part_index| multipart_part_upload_url(part_index + 1) end end |
#multipart_upload_hash ⇒ Object
55 56 57 58 59 60 61 62 63 64 |
# File 'lib/object_storage/direct_upload.rb', line 55 def multipart_upload_hash return unless requires_multipart_upload? { PartSize: rounded_multipart_part_size, PartURLs: multipart_part_urls, CompleteURL: multipart_complete_url, AbortURL: multipart_abort_url } end |
#provider ⇒ Object
152 153 154 |
# File 'lib/object_storage/direct_upload.rb', line 152 def provider credentials[:provider].to_s end |
#store_url ⇒ Object
171 172 173 |
# File 'lib/object_storage/direct_upload.rb', line 171 def store_url connection.put_object_url(bucket_name, object_name, expire_at, ) end |
#to_hash ⇒ Object
42 43 44 45 46 47 48 49 50 51 52 53 |
# File 'lib/object_storage/direct_upload.rb', line 42 def to_hash { Timeout: TIMEOUT, GetURL: get_url, StoreURL: store_url, DeleteURL: delete_url, SkipDelete: skip_delete, MultipartUpload: multipart_upload_hash, CustomPutHeaders: true, PutHeaders: }.merge(workhorse_client_hash).compact end |
#use_workhorse_google_client? ⇒ Boolean
143 144 145 146 147 148 149 150 |
# File 'lib/object_storage/direct_upload.rb', line 143 def use_workhorse_google_client? return false unless config.consolidated_settings? return true if credentials[:google_application_default] return true if credentials[:google_json_key_location] return true if credentials[:google_json_key_string] false end |
#use_workhorse_s3_client? ⇒ Boolean
135 136 137 138 139 140 141 |
# File 'lib/object_storage/direct_upload.rb', line 135 def use_workhorse_s3_client? return false unless config.use_iam_profile? || config.consolidated_settings? # The Golang AWS SDK does not support V2 signatures return false unless credentials.fetch(:aws_signature_version, 4).to_i >= 4 true end |
#workhorse_aws_hash ⇒ Object
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
# File 'lib/object_storage/direct_upload.rb', line 78 def workhorse_aws_hash { UseWorkhorseClient: use_workhorse_s3_client?, RemoteTempObjectID: object_name, ObjectStorage: { Provider: 'AWS', S3Config: { Bucket: bucket_name, Region: credentials[:region] || ::Fog::AWS::Storage::DEFAULT_REGION, Endpoint: credentials[:endpoint], PathStyle: config.use_path_style?, UseIamProfile: config.use_iam_profile?, ServerSideEncryption: config.server_side_encryption, SSEKMSKeyID: config.server_side_encryption_kms_key_id }.compact } } end |
#workhorse_azure_hash ⇒ Object
97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
# File 'lib/object_storage/direct_upload.rb', line 97 def workhorse_azure_hash { # Azure requires Workhorse client because direct uploads can't # use pre-signed URLs without buffering the whole file to disk. UseWorkhorseClient: true, RemoteTempObjectID: object_name, ObjectStorage: { Provider: 'AzureRM', GoCloudConfig: { URL: azure_gocloud_url } } } end |
#workhorse_client_hash ⇒ Object
66 67 68 69 70 71 72 73 74 75 76 |
# File 'lib/object_storage/direct_upload.rb', line 66 def workhorse_client_hash if config.aws? workhorse_aws_hash elsif config.azure? workhorse_azure_hash elsif Feature.enabled?(:workhorse_google_client) && config.google? workhorse_google_hash else {} end end |
#workhorse_google_hash ⇒ Object
118 119 120 121 122 123 124 125 126 127 128 129 |
# File 'lib/object_storage/direct_upload.rb', line 118 def workhorse_google_hash { UseWorkhorseClient: use_workhorse_google_client?, RemoteTempObjectID: object_name, ObjectStorage: { Provider: 'Google', GoCloudConfig: { URL: google_gocloud_url } } } end |