Class: FileStore::S3Store

Inherits:
BaseStore show all
Defined in:
lib/file_store/s3_store.rb

Instance Method Summary collapse

Methods inherited from BaseStore

#cache_file, #download, #download!, #download_safe, #get_cache_path_for, #get_from_cache, #get_path_for, #get_path_for_optimized_image, #get_path_for_upload, #internal?, #relative_base_url, #remove_optimized_image, #remove_upload, temporary_upload_path, #upload_path

Constructor Details

#initialize(s3_helper = nil) ⇒ S3Store

Returns a new instance of S3Store.



19
20
21
# File 'lib/file_store/s3_store.rb', line 19

def initialize(s3_helper = nil)
  @s3_helper = s3_helper
end

Instance Method Details

#absolute_base_urlObject



188
189
190
# File 'lib/file_store/s3_store.rb', line 188

def absolute_base_url
  @absolute_base_url ||= SiteSetting.Upload.absolute_base_url
end

#avatar_template(avatar, user_id) ⇒ Object



288
289
290
# File 'lib/file_store/s3_store.rb', line 288

def avatar_template(avatar, user_id)
  UserAvatar.external_avatar_url(user_id, avatar.upload_id, avatar.width)
end

#cache_avatar(avatar, user_id) ⇒ Object



282
283
284
285
286
# File 'lib/file_store/s3_store.rb', line 282

def cache_avatar(avatar, user_id)
  source = avatar.url.sub(absolute_base_url + "/", "")
  destination = avatar_template(avatar, user_id).sub(absolute_base_url + "/", "")
  s3_helper.copy(source, destination)
end

#cdn_url(url) ⇒ Object



236
237
238
239
240
241
242
243
244
# File 'lib/file_store/s3_store.rb', line 236

def cdn_url(url)
  return url if SiteSetting.Upload.s3_cdn_url.blank?
  schema = url[%r{\A(https?:)?//}, 1]
  folder = s3_bucket_folder_path.nil? ? "" : "#{s3_bucket_folder_path}/"
  url.sub(
    File.join("#{schema}#{absolute_base_url}", folder),
    File.join(SiteSetting.Upload.s3_cdn_url, "/"),
  )
end

#copy_file(url, source, destination) ⇒ Object



136
137
138
139
# File 'lib/file_store/s3_store.rb', line 136

def copy_file(url, source, destination)
  return unless has_been_uploaded?(url)
  s3_helper.copy(source, destination)
end

#copy_from(source_path) ⇒ Object



341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
# File 'lib/file_store/s3_store.rb', line 341

def copy_from(source_path)
  local_store = FileStore::LocalStore.new
  public_upload_path = File.join(local_store.public_dir, local_store.upload_path)

  # The migration to S3 and lots of other code expects files to exist in public/uploads,
  # so lets move them there before executing the migration.
  if public_upload_path != source_path
    if Dir.exist?(public_upload_path)
      old_upload_path = "#{public_upload_path}_#{SecureRandom.hex}"
      FileUtils.mv(public_upload_path, old_upload_path)
    end
  end

  FileUtils.mkdir_p(File.expand_path("..", public_upload_path))
  FileUtils.symlink(source_path, public_upload_path)

  FileStore::ToS3Migration.new(
    s3_options: FileStore::ToS3Migration.s3_options_from_site_settings,
    migrate_to_multisite: Rails.configuration.multisite,
  ).migrate
ensure
  FileUtils.rm(public_upload_path) if File.symlink?(public_upload_path)
  FileUtils.mv(old_upload_path, public_upload_path) if old_upload_path
end

#create_multipart(file_name, content_type, metadata: {}) ⇒ Object



366
367
368
369
# File 'lib/file_store/s3_store.rb', line 366

def create_multipart(file_name, content_type, metadata: {})
  key = temporary_upload_path(file_name)
  s3_helper.create_multipart(key, content_type, metadata: )
end

#delete_file(path) ⇒ Object



124
125
126
127
128
# File 'lib/file_store/s3_store.rb', line 124

def delete_file(path)
  # delete the object outright without moving to tombstone,
  # not recommended for most use cases
  s3_helper.delete_object(path)
end

#download_file(upload, destination_path) ⇒ Object



337
338
339
# File 'lib/file_store/s3_store.rb', line 337

def download_file(upload, destination_path)
  s3_helper.download_file(get_upload_key(upload), destination_path)
end

#download_url(upload) ⇒ Object



212
213
214
215
# File 'lib/file_store/s3_store.rb', line 212

def download_url(upload)
  return unless upload
  "#{upload.short_path}?dl=1"
end

#external?Boolean

Returns:

  • (Boolean)


200
201
202
# File 'lib/file_store/s3_store.rb', line 200

def external?
  true
end

#has_been_uploaded?(url) ⇒ Boolean

Returns:

  • (Boolean)


141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
# File 'lib/file_store/s3_store.rb', line 141

def has_been_uploaded?(url)
  return false if url.blank?

  begin
    parsed_url = URI.parse(UrlHelper.encode(url))
  rescue StandardError
    # There are many exceptions possible here including Addressable::URI:: exceptions
    # and URI:: exceptions, catch all may seem wide, but it makes no sense to raise ever
    # on an invalid url here
    return false
  end

  base_hostname = URI.parse(absolute_base_url).hostname
  if url[base_hostname]
    # if the hostnames match it means the upload is in the same
    # bucket on s3. however, the bucket folder path may differ in
    # some cases, and we do not want to assume the url is uploaded
    # here. e.g. the path of the current site could be /prod and the
    # other site could be /staging
    if s3_bucket_folder_path.present?
      return parsed_url.path.starts_with?("/#{s3_bucket_folder_path}")
    else
      return true
    end
    return false
  end

  return false if SiteSetting.Upload.s3_cdn_url.blank?

  s3_cdn_url = URI.parse(SiteSetting.Upload.s3_cdn_url || "")
  cdn_hostname = s3_cdn_url.hostname

  if cdn_hostname.presence && url[cdn_hostname] &&
       (s3_cdn_url.path.blank? || parsed_url.path.starts_with?(s3_cdn_url.path))
    return true
  end
  false
end

#list_missing_uploads(skip_optimized: false) ⇒ Object



299
300
301
302
303
304
305
306
307
308
# File 'lib/file_store/s3_store.rb', line 299

def list_missing_uploads(skip_optimized: false)
  if SiteSetting.enable_s3_inventory
    require "s3_inventory"
    S3Inventory.new(s3_helper, :upload).backfill_etags_and_list_missing
    S3Inventory.new(s3_helper, :optimized).backfill_etags_and_list_missing unless skip_optimized
  else
    list_missing(Upload.by_users, "original/")
    list_missing(OptimizedImage, "optimized/") unless skip_optimized
  end
end

#move_existing_stored_upload(existing_external_upload_key:, upload: nil, content_type: nil) ⇒ Object



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# File 'lib/file_store/s3_store.rb', line 46

def move_existing_stored_upload(existing_external_upload_key:, upload: nil, content_type: nil)
  upload.url = nil
  path = get_path_for_upload(upload)
  url, upload.etag =
    store_file(
      nil,
      path,
      filename: upload.original_filename,
      content_type: content_type,
      cache_locally: false,
      private_acl: upload.secure?,
      move_existing: true,
      existing_external_upload_key: existing_external_upload_key,
    )
  url
end

#multisite_tombstone_prefixObject



208
209
210
# File 'lib/file_store/s3_store.rb', line 208

def multisite_tombstone_prefix
  File.join("uploads", "tombstone", RailsMultisite::ConnectionManagement.current_db, "/")
end

#object_from_path(path) ⇒ Object



278
279
280
# File 'lib/file_store/s3_store.rb', line 278

def object_from_path(path)
  s3_helper.object(path)
end

#path_for(upload) ⇒ Object



217
218
219
220
# File 'lib/file_store/s3_store.rb', line 217

def path_for(upload)
  url = upload&.url
  FileStore::LocalStore.new.path_for(upload) if url && url[%r{\A/[^/]}]
end

#purge_tombstone(grace_period) ⇒ Object



204
205
206
# File 'lib/file_store/s3_store.rb', line 204

def purge_tombstone(grace_period)
  s3_helper.update_tombstone_lifecycle(grace_period)
end

#remove_file(url, path) ⇒ Object



130
131
132
133
134
# File 'lib/file_store/s3_store.rb', line 130

def remove_file(url, path)
  return unless has_been_uploaded?(url)
  # copy the removed file to tombstone
  s3_helper.remove(path, true)
end

#s3_bucketObject



292
293
294
295
296
297
# File 'lib/file_store/s3_store.rb', line 292

def s3_bucket
  if SiteSetting.Upload.s3_upload_bucket.blank?
    raise Discourse::SiteSettingMissing.new("s3_upload_bucket")
  end
  SiteSetting.Upload.s3_upload_bucket.downcase
end

#s3_bucket_folder_pathObject



180
181
182
# File 'lib/file_store/s3_store.rb', line 180

def s3_bucket_folder_path
  S3Helper.get_bucket_and_folder_path(s3_bucket)[1]
end

#s3_bucket_nameObject



184
185
186
# File 'lib/file_store/s3_store.rb', line 184

def s3_bucket_name
  S3Helper.get_bucket_and_folder_path(s3_bucket)[0]
end

#s3_helperObject



23
24
25
26
27
28
29
# File 'lib/file_store/s3_store.rb', line 23

def s3_helper
  @s3_helper ||=
    S3Helper.new(
      s3_bucket,
      Rails.configuration.multisite ? multisite_tombstone_prefix : TOMBSTONE_PREFIX,
    )
end

#s3_upload_hostObject



192
193
194
195
196
197
198
# File 'lib/file_store/s3_store.rb', line 192

def s3_upload_host
  if SiteSetting.Upload.s3_cdn_url.present?
    SiteSetting.Upload.s3_cdn_url
  else
    "https:#{absolute_base_url}"
  end
end

#signed_request_for_temporary_upload(file_name, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, metadata: {}) ⇒ Object



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
# File 'lib/file_store/s3_store.rb', line 255

def signed_request_for_temporary_upload(
  file_name,
  expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS,
  metadata: {}
)
  key = temporary_upload_path(file_name)
  s3_helper.presigned_request(
    key,
    method: :put_object,
    expires_in: expires_in,
    opts: {
      metadata: ,
      acl: SiteSetting.s3_use_acls ? "private" : nil,
    },
  )
end

#signed_url_for_path(path, expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds, force_download: false) ⇒ Object



246
247
248
249
250
251
252
253
# File 'lib/file_store/s3_store.rb', line 246

def signed_url_for_path(
  path,
  expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds,
  force_download: false
)
  key = path.sub(absolute_base_url + "/", "")
  presigned_get_url(key, expires_in: expires_in, force_download: force_download)
end

#store_file(file, path, opts = {}) ⇒ Object

File is an actual Tempfile on disk

An existing_external_upload_key is given for cases where move_existing is specified. This is an object already uploaded directly to S3 that we are now moving to its final resting place with the correct sha and key.

options

- filename
- content_type
- cache_locally
- move_existing
- existing_external_upload_key


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# File 'lib/file_store/s3_store.rb', line 83

def store_file(file, path, opts = {})
  path = path.dup

  filename = opts[:filename].presence || File.basename(path)
  # cache file locally when needed
  cache_file(file, File.basename(path)) if opts[:cache_locally]
  options = {
    acl: SiteSetting.s3_use_acls ? (opts[:private_acl] ? "private" : "public-read") : nil,
    cache_control: "max-age=31556952, public, immutable",
    content_type:
      opts[:content_type].presence || MiniMime.lookup_by_filename(filename)&.content_type,
  }

  # add a "content disposition: attachment" header with the original
  # filename for everything but safe images (not SVG). audio and video will
  # still stream correctly in HTML players, and when a direct link is
  # provided to any file but an image it will download correctly in the
  # browser.
  if !FileHelper.is_inline_image?(filename)
    options[:content_disposition] = ActionDispatch::Http::ContentDisposition.format(
      disposition: "attachment",
      filename: filename,
    )
  end

  path.prepend(File.join(upload_path, "/")) if Rails.configuration.multisite

  # if this fails, it will throw an exception
  if opts[:move_existing] && opts[:existing_external_upload_key]
    original_path = opts[:existing_external_upload_key]
    options[:apply_metadata_to_destination] = true
    path, etag = s3_helper.copy(original_path, path, options: options)
    delete_file(original_path)
  else
    path, etag = s3_helper.upload(file, path, options)
  end

  # return the upload url and etag
  [File.join(absolute_base_url, path), etag]
end

#store_optimized_image(file, optimized_image, content_type = nil, secure: false) ⇒ Object



63
64
65
66
67
68
69
# File 'lib/file_store/s3_store.rb', line 63

def store_optimized_image(file, optimized_image, content_type = nil, secure: false)
  optimized_image.url = nil
  path = get_path_for_optimized_image(optimized_image)
  url, optimized_image.etag =
    store_file(file, path, content_type: content_type, private_acl: secure)
  url
end

#store_upload(file, upload, content_type = nil) ⇒ Object



31
32
33
34
35
36
37
38
39
40
41
42
43
44
# File 'lib/file_store/s3_store.rb', line 31

def store_upload(file, upload, content_type = nil)
  upload.url = nil
  path = get_path_for_upload(upload)
  url, upload.etag =
    store_file(
      file,
      path,
      filename: upload.original_filename,
      content_type: content_type,
      cache_locally: true,
      private_acl: upload.secure?,
    )
  url
end

#temporary_upload_path(file_name) ⇒ Object



272
273
274
275
276
# File 'lib/file_store/s3_store.rb', line 272

def temporary_upload_path(file_name)
  folder_prefix =
    s3_bucket_folder_path.nil? ? upload_path : File.join(s3_bucket_folder_path, upload_path)
  FileStore::BaseStore.temporary_upload_path(file_name, folder_prefix: folder_prefix)
end

#update_optimized_image_acl(optimized_image, secure: false) ⇒ Object



331
332
333
334
335
# File 'lib/file_store/s3_store.rb', line 331

def update_optimized_image_acl(optimized_image, secure: false)
  optimized_image_key = get_path_for_optimized_image(optimized_image)
  optimized_image_key.prepend(File.join(upload_path, "/")) if Rails.configuration.multisite
  update_ACL(optimized_image_key, secure)
end

#update_upload_ACL(upload, optimized_images_preloaded: false) ⇒ Object



310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
# File 'lib/file_store/s3_store.rb', line 310

def update_upload_ACL(upload, optimized_images_preloaded: false)
  key = get_upload_key(upload)
  update_ACL(key, upload.secure?)

  # If we do find_each when the images have already been preloaded with
  # includes(:optimized_images), then the optimized_images are fetched
  # from the database again, negating the preloading if this operation
  # is done on a large amount of uploads at once (see Jobs::SyncAclsForUploads)
  if optimized_images_preloaded
    upload.optimized_images.each do |optimized_image|
      update_optimized_image_acl(optimized_image, secure: upload.secure)
    end
  else
    upload.optimized_images.find_each do |optimized_image|
      update_optimized_image_acl(optimized_image, secure: upload.secure)
    end
  end

  true
end

#url_for(upload, force_download: false) ⇒ Object



222
223
224
225
226
227
228
229
230
231
232
233
234
# File 'lib/file_store/s3_store.rb', line 222

def url_for(upload, force_download: false)
  if upload.secure? || force_download
    presigned_get_url(
      get_upload_key(upload),
      force_download: force_download,
      filename: upload.original_filename,
    )
  elsif SiteSetting.s3_use_cdn_url_for_all_uploads
    cdn_url(upload.url)
  else
    upload.url
  end
end