Class: UploadCreator

Inherits:
Object
  • Object
show all
Defined in:
lib/upload_creator.rb

Constant Summary collapse

MIN_CONVERT_TO_JPEG_BYTES_SAVED = 
75_000
MIN_CONVERT_TO_JPEG_SAVING_RATIO = 
0.70
MAX_CONVERT_FORMAT_SECONDS = 
20
MAX_FIX_ORIENTATION_TIME = 
5

Instance Method Summary collapse

Constructor Details

#initialize(file, filename, opts = {}) ⇒ UploadCreator

Available options

- type (string)
- origin (string)
- for_group_message (boolean)
- for_theme (boolean)
- for_private_message (boolean)
- pasted (boolean)
- for_export (boolean)
- for_gravatar (boolean)
- skip_validations (boolean)



44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/upload_creator.rb', line 44

def initialize(file, filename, opts = {})
  @file = file
  @filename = (filename || "").gsub(/[^[:print:]]/, "")
  @upload = Upload.new(original_filename: @filename, filesize: 0)
  @opts = opts
  @filesize = @opts[:filesize] if @opts[:external_upload_too_big]
  @opts[:validate] = (
    if opts[:skip_validations].present?
      !ActiveRecord::Type::Boolean.new.cast(opts[:skip_validations])
    else
      true
    end
  )
end

Instance Method Details

#add_metadata! ⇒ Object 



640
641
642
643
644
645
646
647
 
# File 'lib/upload_creator.rb', line 640

def add_metadata!
  @upload.for_private_message = true if @opts[:for_private_message]
  @upload.for_group_message = true if @opts[:for_group_message]
  @upload.for_theme = true if @opts[:for_theme]
  @upload.for_export = true if @opts[:for_export]
  @upload.for_site_setting = true if @opts[:for_site_setting]
  @upload.for_gravatar = true if @opts[:for_gravatar]
end

#clean_svg! ⇒ Object 



499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
 
# File 'lib/upload_creator.rb', line 499

def clean_svg!
  doc = Nokogiri.XML(@file)
  doc.xpath(svg_allowlist_xpath).remove
  doc.xpath("//@*[starts-with(name(), 'on')]").remove
  doc
    .css("use")
    .each do |use_el|
      if use_el.attr("href")
        use_el.remove_attribute("href") unless use_el.attr("href").starts_with?("#")
      end
      use_el.remove_attribute("xlink:href")
    end
  File.write(@file.path, doc.to_s)
  @file.rewind
end

#convert_favicon_to_png! ⇒ Object 



327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/upload_creator.rb', line 327

def convert_favicon_to_png!
  png_tempfile = Tempfile.new(%w[image .png])

  from = @file.path
  to = png_tempfile.path

  OptimizedImage.ensure_safe_paths!(from, to)

  from = OptimizedImage.prepend_decoder!(from, nil, filename: "image.#{@image_info.type}")
  to = OptimizedImage.prepend_decoder!(to)

  from = "#{from}[-1]" # We only want the last(largest) image of the .ico file

  opts = { flatten: false } # Preserve transparency

  begin
    execute_convert(from, to, opts)
  rescue StandardError
    # retry with debugging enabled
    execute_convert(from, to, opts.merge(debug: true))
  end

  @file.respond_to?(:close!) ? @file.close! : @file.close
  @file = png_tempfile
  extract_image_info!
end

#convert_heif! ⇒ Object 



401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
 
# File 'lib/upload_creator.rb', line 401

def convert_heif!
  jpeg_tempfile = Tempfile.new(%w[image .jpg])
  from = @file.path
  to = jpeg_tempfile.path
  OptimizedImage.ensure_safe_paths!(from, to)

  begin
    execute_convert(from, to)
  rescue StandardError
    # retry with debugging enabled
    execute_convert(from, to, { debug: true })
  end

  @file.respond_to?(:close!) ? @file.close! : @file.close
  @file = jpeg_tempfile
  extract_image_info!
end

#convert_heif_to_jpeg? ⇒ Boolean

Returns:

  • (Boolean) 


397
398
399
 
# File 'lib/upload_creator.rb', line 397

def convert_heif_to_jpeg?
  File.extname(@filename).downcase.match?(/\.hei(f|c)\z/)
end

#convert_png_to_jpeg? ⇒ Boolean

Returns:

  • (Boolean) 


317
318
319
320
321
322
 
# File 'lib/upload_creator.rb', line 317

def convert_png_to_jpeg?
  return false unless @image_info.type == :png
  return true if @opts[:pasted]
  return false if SiteSetting.png_to_jpg_quality == 100
  pixels > MIN_PIXELS_TO_CONVERT_TO_JPEG
end

#convert_to_jpeg! ⇒ Object 



354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
 
# File 'lib/upload_creator.rb', line 354

def convert_to_jpeg!
  return if @opts[:for_site_setting]
  return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED

  jpeg_tempfile = Tempfile.new(%w[image .jpg])

  from = @file.path
  to = jpeg_tempfile.path

  OptimizedImage.ensure_safe_paths!(from, to)

  from = OptimizedImage.prepend_decoder!(from, nil, filename: "image.#{@image_info.type}")
  to = OptimizedImage.prepend_decoder!(to)

  opts = {}
  desired_quality = [
    SiteSetting.png_to_jpg_quality,
    SiteSetting.recompress_original_jpg_quality,
  ].compact.min
  target_quality = @upload.target_image_quality(from, desired_quality)
  opts = { quality: target_quality } if target_quality

  begin
    execute_convert(from, to, opts)
  rescue StandardError
    # retry with debugging enabled
    execute_convert(from, to, opts.merge(debug: true))
  end

  new_size = File.size(jpeg_tempfile.path)

  keep_jpeg = new_size < filesize * MIN_CONVERT_TO_JPEG_SAVING_RATIO
  keep_jpeg &&= (filesize - new_size) > MIN_CONVERT_TO_JPEG_BYTES_SAVED

  if keep_jpeg
    @file.respond_to?(:close!) ? @file.close! : @file.close
    @file = jpeg_tempfile
    extract_image_info!
  else
    jpeg_tempfile.close!
  end
end

#create_for(user_id) ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
 
# File 'lib/upload_creator.rb', line 59

def create_for(user_id)
  if filesize <= 0
    @upload.errors.add(:base, I18n.t("upload.empty"))
    return @upload
  end

  @image_info =
    begin
      FastImage.new(@file)
    rescue StandardError
      nil
    end
  is_image = FileHelper.is_supported_image?(@filename)
  is_image ||= @image_info && FileHelper.is_supported_image?("test.#{@image_info.type}")
  is_image = false if @opts[:for_theme]
  is_thumbnail = SiteSetting.video_thumbnails_enabled && @opts[:type] == "thumbnail"

  # If this is present then it means we are creating an upload record from
  # an external_upload_stub and the file is > ExternalUploadManager::DOWNLOAD_LIMIT,
  # so we have not downloaded it to a tempfile. no modifications can be made to the
  # file in this case because it does not exist; we simply move it to its new location
  # in S3
  #
  # FIXME: I've added a bunch of external_upload_too_big checks littered
  # throughout the UploadCreator code. It would be better to have two separate
  # classes with shared methods, rather than doing all these checks all over the
  # place. Needs a refactor.
  external_upload_too_big = @opts[:external_upload_too_big]
  sha1_before_changes = Upload.generate_digest(@file) if @file

  DistributedMutex.synchronize("upload_#{user_id}_#{@filename}") do
    # We need to convert HEIFs early because FastImage does not consider them as images
    if convert_heif_to_jpeg? && !external_upload_too_big
      convert_heif!
      is_image = FileHelper.is_supported_image?("test.#{@image_info.type}")
    end

    if is_image && !external_upload_too_big
      extract_image_info!
      return @upload if @upload.errors.present?

      if @image_info.type == :svg
        clean_svg!
      elsif @image_info.type == :ico
        convert_favicon_to_png!
      elsif !Rails.env.test? || @opts[:force_optimize]
        convert_to_jpeg! if convert_png_to_jpeg? || should_alter_quality?
        fix_orientation! if should_fix_orientation?
        crop! if should_crop?
        optimize! if should_optimize?
        downsize! if should_downsize?
        return @upload if is_still_too_big?
      end

      # conversion may have switched the type
      image_type = @image_info.type.to_s
    end

    # compute the sha of the file and generate a unique hash
    # which is only used for secure uploads
    sha1 = Upload.generate_digest(@file) if !external_upload_too_big
    unique_hash = generate_fake_sha1_hash if SiteSetting.secure_uploads ||
      external_upload_too_big || is_thumbnail

    # we do not check for duplicate uploads if secure uploads is
    # enabled because we use a unique access hash to differentiate
    # between uploads instead of the sha1, and to get around various
    # access/permission issues for uploads
    # We do not check for duplicate uploads for video thumbnails because
    # their filename needs to match with their corresponding video. This also
    # enables rebuilding the html on a topic to regenerate a thumbnail.
    if !SiteSetting.secure_uploads && !external_upload_too_big && !is_thumbnail
      # do we already have that upload?
      @upload = Upload.find_by(sha1: sha1)

      # make sure the previous upload has not failed
      if @upload && @upload.url.blank?
        @upload.destroy
        @upload = nil
      end

      # return the previous upload if any
      if @upload
        add_metadata!
        UserUpload.find_or_create_by!(user_id: user_id, upload_id: @upload.id) if user_id
        return @upload
      end
    end

    fixed_original_filename = nil

    if is_image && !external_upload_too_big
      current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg")
      expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg")

      # we have to correct original filename here, no choice
      # otherwise validation will fail and we can not save
      # TODO decide if we only run the validation on the extension
      if current_extension != expected_extension
        basename = File.basename(@filename, current_extension).presence || "image"
        fixed_original_filename = "#{basename}#{expected_extension}"
      end
    end

    # create the upload otherwise
    @upload = Upload.new
    @upload.user_id = user_id
    @upload.original_filename = fixed_original_filename || @filename
    @upload.filesize = filesize
    @upload.sha1 =
      (
        if (SiteSetting.secure_uploads? || external_upload_too_big || is_thumbnail)
          unique_hash
        else
          sha1
        end
      )
    @upload.original_sha1 = SiteSetting.secure_uploads? ? sha1 : nil
    @upload.url = ""
    @upload.origin = @opts[:origin][0...1000] if @opts[:origin]
    @upload.extension = image_type || File.extname(@filename)[1..10]

    if is_image && !external_upload_too_big
      if @image_info.type.to_s == "svg"
        w, h = [0, 0]

        # identify can behave differently depending on how it's compiled and
        # what programs (e.g. inkscape) are installed on your system.
        # 'MSVG:' forces ImageMagick to use internal routines and behave
        # consistently whether it's running from our docker container or not
        begin
          w, h =
            Discourse::Utils
              .execute_command(
                "identify",
                "-ping",
                "-format",
                "%w %h",
                "MSVG:#{@file.path}",
                timeout: Upload::MAX_IDENTIFY_SECONDS,
              )
              .split(" ")
              .map(&:to_i)
        rescue StandardError
          # use default 0, 0
        end
      else
        w, h = @image_info.size
      end

      @upload.thumbnail_width, @upload.thumbnail_height = ImageSizer.resize(w, h)
      @upload.width, @upload.height = w, h
      @upload.animated = animated?
      @upload.calculate_dominant_color!(@file.path)
    end

    add_metadata!

    if SiteSetting.secure_uploads
      secure, reason =
        UploadSecurity.new(@upload, @opts.merge(creating: true)).should_be_secure_with_reason
      attrs = @upload.secure_params(secure, reason, "upload creator")
      @upload.assign_attributes(attrs)
    end

    # Callbacks using this event to validate the upload or the file must add errors to the
    # upload errors object.
    #
    # Can't do any validation of the file if external_upload_too_big because we don't have
    # the actual file.
    if !external_upload_too_big
      DiscourseEvent.trigger(:before_upload_creation, @file, is_image, @upload, @opts[:validate])
    end
    return @upload unless @upload.errors.empty? && @upload.save(validate: @opts[:validate])

    should_move = false
    upload_changed =
      if external_upload_too_big
        false
      else
        Upload.generate_digest(@file) != sha1_before_changes
      end

    store = Discourse.store

    if @opts[:existing_external_upload_key] && store.external?
      should_move = external_upload_too_big || !upload_changed
    end

    if should_move
      # move the file in the store instead of reuploading
      url =
        store.move_existing_stored_upload(
          existing_external_upload_key: @opts[:existing_external_upload_key],
          upload: @upload,
        )
    else
      # store the file and update its url
      File.open(@file.path) { |f| url = store.store_upload(f, @upload) }
      if @opts[:existing_external_upload_key]
        store.delete_file(@opts[:existing_external_upload_key])
      end
    end

    if url.present?
      @upload.url = url
      @upload.save!(validate: @opts[:validate])
    else
      @upload.errors.add(
        :url,
        I18n.t("upload.store_failure", upload_id: @upload.id, user_id: user_id),
      )
    end

    if @upload.errors.empty? && is_image && @opts[:type] == "avatar" && @upload.extension != "svg"
      Jobs.enqueue(:create_avatar_thumbnails, upload_id: @upload.id)
    end

    if @upload.errors.empty?
      UserUpload.find_or_create_by!(user_id: user_id, upload_id: @upload.id) if user_id
    end

    @upload
  end
ensure
  if @file
    @file.respond_to?(:close!) ? @file.close! : @file.close
  end
end

#crop! ⇒ Object 



547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
 
# File 'lib/upload_creator.rb', line 547

def crop!
  max_pixel_ratio = Discourse::PIXEL_RATIOS.max
  filename_with_correct_ext = "image.#{@image_info.type}"

  case @opts[:type]
  when "avatar"
    width = height = Discourse.avatar_sizes.max
    OptimizedImage.resize(
      @file.path,
      @file.path,
      width,
      height,
      filename: filename_with_correct_ext,
    )
  when "profile_background"
    max_width = 850 * max_pixel_ratio
    width, height =
      ImageSizer.resize(
        @image_info.size[0],
        @image_info.size[1],
        max_width: max_width,
        max_height: max_width,
      )
    OptimizedImage.downsize(
      @file.path,
      @file.path,
      "#{width}x#{height}\>",
      filename: filename_with_correct_ext,
    )
  when "card_background"
    max_width = 590 * max_pixel_ratio
    width, height =
      ImageSizer.resize(
        @image_info.size[0],
        @image_info.size[1],
        max_width: max_width,
        max_height: max_width,
      )
    OptimizedImage.downsize(
      @file.path,
      @file.path,
      "#{width}x#{height}\>",
      filename: filename_with_correct_ext,
    )
  when "custom_emoji"
    OptimizedImage.downsize(
      @file.path,
      @file.path,
      "100x100\>",
      filename: filename_with_correct_ext,
    )
  end

  extract_image_info!
end

#downsize! ⇒ Object 



452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
 
# File 'lib/upload_creator.rb', line 452

def downsize!
  3.times do
    original_size = filesize
    down_tempfile = Tempfile.new(["down", ".#{@image_info.type}"])

    from = @file.path
    to = down_tempfile.path

    OptimizedImage.ensure_safe_paths!(from, to)

    OptimizedImage.downsize(from, to, "50%", scale_image: true, raise_on_error: true)

    @file.respond_to?(:close!) ? @file.close! : @file.close
    @file = down_tempfile

    extract_image_info!

    return if filesize >= original_size || pixels == 0 || !should_downsize?
  end
rescue StandardError
  @upload.errors.add(:base, I18n.t("upload.optimize_failure_message"))
end

#execute_convert(from, to, opts = {}) ⇒ Object 



420
421
422
423
424
425
426
427
428
429
430
431
432
 
# File 'lib/upload_creator.rb', line 420

def execute_convert(from, to, opts = {})
  command = ["convert", from, "-auto-orient", "-background", "white", "-interlace", "none"]
  command << "-flatten" unless opts[:flatten] == false
  command << "-debug" << "all" if opts[:debug]
  command << "-quality" << opts[:quality].to_s if opts[:quality]
  command << to

  Discourse::Utils.execute_command(
    *command,
    failure_message: I18n.t("upload.png_to_jpg_conversion_failure_message"),
    timeout: MAX_CONVERT_FORMAT_SECONDS,
  )
end

#extract_image_info! ⇒ Object 



289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/upload_creator.rb', line 289

def extract_image_info!
  @image_info =
    begin
      FastImage.new(@file)
    rescue StandardError
      nil
    end
  @file.rewind

  if @image_info.nil?
    @upload.errors.add(:base, I18n.t("upload.images.not_supported_or_corrupted"))
  elsif filesize <= 0
    @upload.errors.add(:base, I18n.t("upload.empty"))
  elsif pixels == 0 && @image_info.type.to_s != "svg"
    @upload.errors.add(:base, I18n.t("upload.images.size_not_found"))
  elsif max_image_pixels > 0 && pixels >= max_image_pixels * 2
    @upload.errors.add(
      :base,
      I18n.t(
        "upload.images.larger_than_x_megapixels",
        max_image_megapixels: SiteSetting.max_image_megapixels * 2,
      ),
    )
  end
end

#filesize ⇒ Object 



619
620
621
 
# File 'lib/upload_creator.rb', line 619

def filesize
  @filesize || File.size?(@file.path).to_i
end

#fix_orientation! ⇒ Object 



522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
 
# File 'lib/upload_creator.rb', line 522

def fix_orientation!
  path = @file.path

  OptimizedImage.ensure_safe_paths!(path)
  path = OptimizedImage.prepend_decoder!(path, nil, filename: "image.#{@image_info.type}")

  Discourse::Utils.execute_command(
    "convert",
    path,
    "-auto-orient",
    path,
    timeout: MAX_FIX_ORIENTATION_TIME,
  )

  extract_image_info!
end

#is_still_too_big? ⇒ Boolean

Returns:

  • (Boolean) 


475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
 
# File 'lib/upload_creator.rb', line 475

def is_still_too_big?
  if max_image_pixels > 0 && pixels >= max_image_pixels
    @upload.errors.add(
      :base,
      I18n.t(
        "upload.images.larger_than_x_megapixels",
        max_image_megapixels: SiteSetting.max_image_megapixels,
      ),
    )
    true
  elsif max_image_size > 0 && filesize >= max_image_size
    @upload.errors.add(
      :base,
      I18n.t(
        "upload.images.too_large_humanized",
        max_size: ActiveSupport::NumberHelper.number_to_human_size(max_image_size),
      ),
    )
    true
  else
    false
  end
end

#max_image_pixels ⇒ Object 



627
628
629
 
# File 'lib/upload_creator.rb', line 627

def max_image_pixels
  @max_image_pixels ||= SiteSetting.max_image_megapixels * 1_000_000
end

#max_image_size ⇒ Object 



623
624
625
 
# File 'lib/upload_creator.rb', line 623

def max_image_size
  @max_image_size ||= SiteSetting.max_image_size_kb.kilobytes
end

#optimize! ⇒ Object 



613
614
615
616
617
 
# File 'lib/upload_creator.rb', line 613

def optimize!
  OptimizedImage.ensure_safe_paths!(@file.path)
  FileHelper.optimize_image!(@file.path)
  extract_image_info!
end

#pixels ⇒ Object 



631
632
633
 
# File 'lib/upload_creator.rb', line 631

def pixels
  @image_info.size&.reduce(:*).to_i
end

#should_alter_quality? ⇒ Boolean

Returns:

  • (Boolean) 


434
435
436
437
438
439
440
441
442
443
444
445
446
 
# File 'lib/upload_creator.rb', line 434

def should_alter_quality?
  return false if animated?

  desired_quality =
    (
      if @image_info.type == :png
        SiteSetting.png_to_jpg_quality
      else
        SiteSetting.recompress_original_jpg_quality
      end
    )
  @upload.target_image_quality(@file.path, desired_quality).present?
end

#should_crop? ⇒ Boolean

Returns:

  • (Boolean) 


539
540
541
542
543
544
545
 
# File 'lib/upload_creator.rb', line 539

def should_crop?
  if %w[profile_background card_background custom_emoji].include?(@opts[:type]) && animated?
    return false
  end

  TYPES_TO_CROP.include?(@opts[:type])
end

#should_downsize? ⇒ Boolean

Returns:

  • (Boolean) 


448
449
450
 
# File 'lib/upload_creator.rb', line 448

def should_downsize?
  max_image_size > 0 && filesize >= max_image_size && !animated?
end

#should_fix_orientation? ⇒ Boolean

Returns:

  • (Boolean) 


515
516
517
518
519
 
# File 'lib/upload_creator.rb', line 515

def should_fix_orientation?
  # orientation is between 1 and 8, 1 being the default
  # cf. http://www.daveperrett.com/articles/2012/07/28/exif-orientation-handling-is-a-ghetto/
  @image_info.orientation.to_i > 1
end

#should_optimize? ⇒ Boolean

Returns:

  • (Boolean) 


603
604
605
606
607
608
609
610
611
 
# File 'lib/upload_creator.rb', line 603

def should_optimize?
  # GIF is too slow (plus, we'll soon be converting them to MP4)
  # Optimizing SVG is useless
  return false if @file.path =~ /\.(gif|svg)\z/i
  # Safeguard for large PNGs
  return pixels < 2_000_000 if @file.path =~ /\.png/i
  # Everything else is fine!
  true
end

#svg_allowlist_xpath ⇒ Object 



635
636
637
638
 
# File 'lib/upload_creator.rb', line 635

def svg_allowlist_xpath
  @@svg_allowlist_xpath ||=
    "//*[#{ALLOWED_SVG_ELEMENTS.map { |e| "name()!='#{e}'" }.join(" and ")}]"
end