Class: FileStore::S3Store
Constant Summary
collapse
- TOMBSTONE_PREFIX =
"tombstone/"
Constants inherited
from BaseStore
BaseStore::CACHE_DIR, BaseStore::CACHE_MAXIMUM_SIZE, BaseStore::OPTIMIZED_IMAGE_PATH_REGEX, BaseStore::TEMPORARY_UPLOAD_PREFIX, BaseStore::UPLOAD_PATH_REGEX
Instance Method Summary
collapse
-
#absolute_base_url ⇒ Object
-
#avatar_template(avatar, user_id) ⇒ Object
-
#cache_avatar(avatar, user_id) ⇒ Object
-
#cdn_url(url) ⇒ Object
-
#copy_file(url, source, destination) ⇒ Object
-
#copy_from(source_path) ⇒ Object
-
#create_multipart(file_name, content_type, metadata: {}) ⇒ Object
-
#delete_file(path) ⇒ Object
-
#download_file(upload, destination_path) ⇒ Object
-
#download_url(upload) ⇒ Object
-
#external? ⇒ Boolean
-
#has_been_uploaded?(url) ⇒ Boolean
-
#initialize(s3_helper = nil) ⇒ S3Store
constructor
A new instance of S3Store.
-
#list_missing_uploads(skip_optimized: false) ⇒ Object
-
#move_existing_stored_upload(existing_external_upload_key:, upload: nil, content_type: nil) ⇒ Object
-
#multisite_tombstone_prefix ⇒ Object
-
#object_from_path(path) ⇒ Object
-
#path_for(upload) ⇒ Object
-
#purge_tombstone(grace_period) ⇒ Object
-
#remove_file(url, path) ⇒ Object
-
#s3_bucket ⇒ Object
-
#s3_bucket_folder_path ⇒ Object
-
#s3_bucket_name ⇒ Object
-
#s3_helper ⇒ Object
-
#s3_upload_host ⇒ Object
-
#signed_request_for_temporary_upload(file_name, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, metadata: {}) ⇒ Object
-
#signed_url_for_path(path, expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds, force_download: false) ⇒ Object
-
#store_file(file, path, opts = {}) ⇒ Object
File is an actual Tempfile on disk.
-
#store_optimized_image(file, optimized_image, content_type = nil, secure: false) ⇒ Object
-
#store_upload(file, upload, content_type = nil) ⇒ Object
-
#temporary_upload_path(file_name) ⇒ Object
-
#update_optimized_image_acl(optimized_image, secure: false) ⇒ Object
-
#update_upload_ACL(upload, optimized_images_preloaded: false) ⇒ Object
-
#url_for(upload, force_download: false) ⇒ Object
Methods inherited from BaseStore
#cache_file, #download, #download!, #download_safe, #get_cache_path_for, #get_from_cache, #get_path_for, #get_path_for_optimized_image, #get_path_for_upload, #internal?, #relative_base_url, #remove_optimized_image, #remove_upload, temporary_upload_path, #upload_path
Constructor Details
#initialize(s3_helper = nil) ⇒ S3Store
Returns a new instance of S3Store.
19
20
21
|
# File 'lib/file_store/s3_store.rb', line 19
def initialize(s3_helper = nil)
@s3_helper = s3_helper
end
|
Instance Method Details
#absolute_base_url ⇒ Object
188
189
190
|
# File 'lib/file_store/s3_store.rb', line 188
def absolute_base_url
@absolute_base_url ||= SiteSetting.Upload.absolute_base_url
end
|
#avatar_template(avatar, user_id) ⇒ Object
288
289
290
|
# File 'lib/file_store/s3_store.rb', line 288
def avatar_template(avatar, user_id)
UserAvatar.external_avatar_url(user_id, avatar.upload_id, avatar.width)
end
|
#cache_avatar(avatar, user_id) ⇒ Object
282
283
284
285
286
|
# File 'lib/file_store/s3_store.rb', line 282
def cache_avatar(avatar, user_id)
source = avatar.url.sub(absolute_base_url + "/", "")
destination = avatar_template(avatar, user_id).sub(absolute_base_url + "/", "")
s3_helper.copy(source, destination)
end
|
#cdn_url(url) ⇒ Object
236
237
238
239
240
241
242
243
244
|
# File 'lib/file_store/s3_store.rb', line 236
def cdn_url(url)
return url if SiteSetting.Upload.s3_cdn_url.blank?
schema = url[%r{\A(https?:)?//}, 1]
folder = s3_bucket_folder_path.nil? ? "" : "#{s3_bucket_folder_path}/"
url.sub(
File.join("#{schema}#{absolute_base_url}", folder),
File.join(SiteSetting.Upload.s3_cdn_url, "/"),
)
end
|
#copy_file(url, source, destination) ⇒ Object
137
138
139
140
|
# File 'lib/file_store/s3_store.rb', line 137
def copy_file(url, source, destination)
return unless has_been_uploaded?(url)
s3_helper.copy(source, destination)
end
|
#copy_from(source_path) ⇒ Object
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
|
# File 'lib/file_store/s3_store.rb', line 349
def copy_from(source_path)
local_store = FileStore::LocalStore.new
public_upload_path = File.join(local_store.public_dir, local_store.upload_path)
if public_upload_path != source_path
if Dir.exist?(public_upload_path)
old_upload_path = "#{public_upload_path}_#{SecureRandom.hex}"
FileUtils.mv(public_upload_path, old_upload_path)
end
end
FileUtils.mkdir_p(File.expand_path("..", public_upload_path))
FileUtils.symlink(source_path, public_upload_path)
FileStore::ToS3Migration.new(
s3_options: FileStore::ToS3Migration.s3_options_from_site_settings,
migrate_to_multisite: Rails.configuration.multisite,
).migrate
ensure
FileUtils.rm(public_upload_path) if File.symlink?(public_upload_path)
FileUtils.mv(old_upload_path, public_upload_path) if old_upload_path
end
|
#create_multipart(file_name, content_type, metadata: {}) ⇒ Object
374
375
376
377
|
# File 'lib/file_store/s3_store.rb', line 374
def create_multipart(file_name, content_type, metadata: {})
key = temporary_upload_path(file_name)
s3_helper.create_multipart(key, content_type, metadata: metadata)
end
|
#delete_file(path) ⇒ Object
125
126
127
128
129
|
# File 'lib/file_store/s3_store.rb', line 125
def delete_file(path)
s3_helper.delete_object(path)
end
|
#download_file(upload, destination_path) ⇒ Object
345
346
347
|
# File 'lib/file_store/s3_store.rb', line 345
def download_file(upload, destination_path)
s3_helper.download_file(get_upload_key(upload), destination_path)
end
|
#download_url(upload) ⇒ Object
212
213
214
215
|
# File 'lib/file_store/s3_store.rb', line 212
def download_url(upload)
return unless upload
"#{upload.short_path}?dl=1"
end
|
#external? ⇒ Boolean
200
201
202
|
# File 'lib/file_store/s3_store.rb', line 200
def external?
true
end
|
#has_been_uploaded?(url) ⇒ Boolean
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
# File 'lib/file_store/s3_store.rb', line 142
def has_been_uploaded?(url)
return false if url.blank?
begin
parsed_url = URI.parse(UrlHelper.encode(url))
rescue StandardError
return false
end
base_hostname = URI.parse(absolute_base_url).hostname
if url[base_hostname]
if s3_bucket_folder_path.present?
return parsed_url.path.starts_with?("/#{s3_bucket_folder_path}")
else
return true
end
end
return false if SiteSetting.Upload.s3_cdn_url.blank?
s3_cdn_url = URI.parse(SiteSetting.Upload.s3_cdn_url || "")
cdn_hostname = s3_cdn_url.hostname
if cdn_hostname.presence && url[cdn_hostname] &&
(s3_cdn_url.path.blank? || parsed_url.path.starts_with?(s3_cdn_url.path))
return true
end
false
end
|
#list_missing_uploads(skip_optimized: false) ⇒ Object
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
|
# File 'lib/file_store/s3_store.rb', line 299
def list_missing_uploads(skip_optimized: false)
if s3_inventory_bucket = SiteSetting.s3_inventory_bucket
s3_options = {}
if (s3_inventory_bucket_region = SiteSetting.s3_inventory_bucket_region).present?
s3_options[:region] = s3_inventory_bucket_region
end
S3Inventory.new(:upload, s3_inventory_bucket:, s3_options:).backfill_etags_and_list_missing
unless skip_optimized
S3Inventory.new(:optimized, s3_inventory_bucket:).backfill_etags_and_list_missing
end
else
list_missing(Upload.by_users, "original/")
list_missing(OptimizedImage, "optimized/") unless skip_optimized
end
end
|
#move_existing_stored_upload(existing_external_upload_key:, upload: nil, content_type: nil) ⇒ Object
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
# File 'lib/file_store/s3_store.rb', line 47
def move_existing_stored_upload(existing_external_upload_key:, upload: nil, content_type: nil)
upload.url = nil
path = get_path_for_upload(upload)
url, upload.etag =
store_file(
nil,
path,
filename: upload.original_filename,
content_type: content_type,
cache_locally: false,
private_acl: upload.secure?,
move_existing: true,
existing_external_upload_key: existing_external_upload_key,
)
url
end
|
#multisite_tombstone_prefix ⇒ Object
208
209
210
|
# File 'lib/file_store/s3_store.rb', line 208
def multisite_tombstone_prefix
File.join("uploads", "tombstone", RailsMultisite::ConnectionManagement.current_db, "/")
end
|
#object_from_path(path) ⇒ Object
278
279
280
|
# File 'lib/file_store/s3_store.rb', line 278
def object_from_path(path)
s3_helper.object(path)
end
|
#path_for(upload) ⇒ Object
217
218
219
220
|
# File 'lib/file_store/s3_store.rb', line 217
def path_for(upload)
url = upload&.url
FileStore::LocalStore.new.path_for(upload) if url && url[%r{\A/[^/]}]
end
|
#purge_tombstone(grace_period) ⇒ Object
204
205
206
|
# File 'lib/file_store/s3_store.rb', line 204
def purge_tombstone(grace_period)
s3_helper.update_tombstone_lifecycle(grace_period)
end
|
#remove_file(url, path) ⇒ Object
131
132
133
134
135
|
# File 'lib/file_store/s3_store.rb', line 131
def remove_file(url, path)
return unless has_been_uploaded?(url)
s3_helper.remove(path, true)
end
|
#s3_bucket_folder_path ⇒ Object
#s3_helper ⇒ Object
23
24
25
26
27
28
29
30
|
# File 'lib/file_store/s3_store.rb', line 23
def s3_helper
@s3_helper ||=
S3Helper.new(
s3_bucket,
Rails.configuration.multisite ? multisite_tombstone_prefix : TOMBSTONE_PREFIX,
use_accelerate_endpoint: SiteSetting.Upload.enable_s3_transfer_acceleration,
)
end
|
#s3_upload_host ⇒ Object
192
193
194
195
196
197
198
|
# File 'lib/file_store/s3_store.rb', line 192
def s3_upload_host
if SiteSetting.Upload.s3_cdn_url.present?
SiteSetting.Upload.s3_cdn_url
else
"https:#{absolute_base_url}"
end
end
|
#signed_request_for_temporary_upload(file_name, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, metadata: {}) ⇒ Object
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
|
# File 'lib/file_store/s3_store.rb', line 255
def signed_request_for_temporary_upload(
file_name,
expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS,
metadata: {}
)
key = temporary_upload_path(file_name)
s3_helper.presigned_request(
key,
method: :put_object,
expires_in: expires_in,
opts: {
metadata: metadata,
acl: SiteSetting.s3_use_acls ? "private" : nil,
},
)
end
|
#signed_url_for_path(path, expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds, force_download: false) ⇒ Object
246
247
248
249
250
251
252
253
|
# File 'lib/file_store/s3_store.rb', line 246
def signed_url_for_path(
path,
expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds,
force_download: false
)
key = path.sub(absolute_base_url + "/", "")
presigned_get_url(key, expires_in: expires_in, force_download: force_download)
end
|
#store_file(file, path, opts = {}) ⇒ Object
File is an actual Tempfile on disk
An existing_external_upload_key is given for cases where move_existing is specified. This is an object already uploaded directly to S3 that we are now moving to its final resting place with the correct sha and key.
options
- filename
- content_type
- cache_locally
- move_existing
- existing_external_upload_key
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
# File 'lib/file_store/s3_store.rb', line 84
def store_file(file, path, opts = {})
path = path.dup
filename = opts[:filename].presence || File.basename(path)
cache_file(file, File.basename(path)) if opts[:cache_locally]
options = {
acl: SiteSetting.s3_use_acls ? (opts[:private_acl] ? "private" : "public-read") : nil,
cache_control: "max-age=31556952, public, immutable",
content_type:
opts[:content_type].presence || MiniMime.lookup_by_filename(filename)&.content_type,
}
if !FileHelper.is_inline_image?(filename)
options[:content_disposition] = ActionDispatch::Http::ContentDisposition.format(
disposition: "attachment",
filename: filename,
)
end
path.prepend(File.join(upload_path, "/")) if Rails.configuration.multisite
if opts[:move_existing] && opts[:existing_external_upload_key]
original_path = opts[:existing_external_upload_key]
options[:apply_metadata_to_destination] = true
path, etag = s3_helper.copy(original_path, path, options: options)
delete_file(original_path)
else
path, etag = s3_helper.upload(file, path, options)
end
[File.join(absolute_base_url, path), etag]
end
|
#store_optimized_image(file, optimized_image, content_type = nil, secure: false) ⇒ Object
64
65
66
67
68
69
70
|
# File 'lib/file_store/s3_store.rb', line 64
def store_optimized_image(file, optimized_image, content_type = nil, secure: false)
optimized_image.url = nil
path = get_path_for_optimized_image(optimized_image)
url, optimized_image.etag =
store_file(file, path, content_type: content_type, private_acl: secure)
url
end
|
#store_upload(file, upload, content_type = nil) ⇒ Object
32
33
34
35
36
37
38
39
40
41
42
43
44
45
|
# File 'lib/file_store/s3_store.rb', line 32
def store_upload(file, upload, content_type = nil)
upload.url = nil
path = get_path_for_upload(upload)
url, upload.etag =
store_file(
file,
path,
filename: upload.original_filename,
content_type: content_type,
cache_locally: true,
private_acl: upload.secure?,
)
url
end
|
#temporary_upload_path(file_name) ⇒ Object
272
273
274
275
276
|
# File 'lib/file_store/s3_store.rb', line 272
def temporary_upload_path(file_name)
folder_prefix =
s3_bucket_folder_path.nil? ? upload_path : File.join(s3_bucket_folder_path, upload_path)
FileStore::BaseStore.temporary_upload_path(file_name, folder_prefix: folder_prefix)
end
|
#update_optimized_image_acl(optimized_image, secure: false) ⇒ Object
339
340
341
342
343
|
# File 'lib/file_store/s3_store.rb', line 339
def update_optimized_image_acl(optimized_image, secure: false)
optimized_image_key = get_path_for_optimized_image(optimized_image)
optimized_image_key.prepend(File.join(upload_path, "/")) if Rails.configuration.multisite
update_ACL(optimized_image_key, secure)
end
|
#update_upload_ACL(upload, optimized_images_preloaded: false) ⇒ Object
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
|
# File 'lib/file_store/s3_store.rb', line 318
def update_upload_ACL(upload, optimized_images_preloaded: false)
key = get_upload_key(upload)
update_ACL(key, upload.secure?)
if optimized_images_preloaded
upload.optimized_images.each do |optimized_image|
update_optimized_image_acl(optimized_image, secure: upload.secure)
end
else
upload.optimized_images.find_each do |optimized_image|
update_optimized_image_acl(optimized_image, secure: upload.secure)
end
end
true
end
|
#url_for(upload, force_download: false) ⇒ Object
222
223
224
225
226
227
228
229
230
231
232
233
234
|
# File 'lib/file_store/s3_store.rb', line 222
def url_for(upload, force_download: false)
if upload.secure? || force_download
presigned_get_url(
get_upload_key(upload),
force_download: force_download,
filename: upload.original_filename,
)
elsif SiteSetting.s3_use_cdn_url_for_all_uploads
cdn_url(upload.url)
else
upload.url
end
end
|