Module: MultipartUploadUtils

Included in:
FilestackClient
Defined in:
lib/filestack/utils/multipart_upload_utils.rb

Overview

Includes all the utility functions for Filestack multipart uploads

Instance Method Summary collapse

Instance Method Details

#create_upload_jobs(apikey, filename, filepath, filesize, start_response, options) ⇒ Array

Create array of jobs for parallel uploading

Parameters:

  • apikey (String) —

    Filestack API key

  • filename (String) —

    Name of incoming file

  • filepath (String) —

    Local path to file

  • filesize (Int) —

    Size of incoming file

  • start_response (Typhoeus::Response) —

    Response body from multipart_start

  • options (Hash) —

    User-defined options for multipart uploads

Returns:

  • (Array) 


78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 78

def create_upload_jobs(apikey, filename, filepath, filesize, start_response, options)
  jobs = []
  part = 1
  seek_point = 0
  while seek_point < filesize
    part_info = {
      seek: seek_point,
      filepath: filepath,
      filename: filename,
      apikey: apikey,
      part: part,
      filesize: filesize,
      uri: start_response['uri'],
      region: start_response['region'],
      upload_id: start_response['upload_id'],
      location_url: start_response['location_url'],
      start_response: start_response,
      options: options,
      store_location: options.nil? ? 's3' : options[:store_location]
    }
    if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
      size = filesize - (seek_point)
    else
      size = FilestackConfig::DEFAULT_CHUNK_SIZE
    end
    part_info[:size] = size
    jobs.push(part_info)
    part += 1
    seek_point += FilestackConfig::DEFAULT_CHUNK_SIZE
  end
  jobs
end

#get_file_info(file) ⇒ Object 



15
16
17
18
19
20
21
22
23
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 15

def get_file_info(file)
  filename = File.basename(file)
  filesize = File.size(file)
  mimetype = MimeMagic.by_magic(File.open(file))
  if mimetype.nil?
    mimetype = 'application/octet-stream'
  end
  [filename, filesize, mimetype.to_s]
end

#multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options, intelligent = false) ⇒ Typhoeus::Response

Send complete call to multipart endpoint

Parameters:

  • apikey (String) —

    Filestack API key

  • filename (String) —

    Name of incoming file

  • filesize (Int) —

    Size of incoming file

  • mimetype (String) —

    Mimetype of incoming file

  • start_response (Typhoeus::Response) —

    Response body from multipart_start

  • security (FilestackSecurity) —

    Security object with policy/signature

  • parts_and_etags (Array) —

    Array of strings defining etags and their associated part numbers

  • options (Hash) —

    User-defined options for multipart uploads

Returns:

  • (Typhoeus::Response) 


192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 192

def multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options, intelligent = false)
  if !intelligent
    data = {
      apikey: apikey,
      uri: start_response['uri'],
      region: start_response['region'],
      upload_id: start_response['upload_id'],
      filename: filename,
      size: filesize,
      mimetype: mimetype,
      parts: parts_and_etags.join(';'),
      store_location: options.nil? ? 's3' : options[:store_location],
      file: Tempfile.new(filename)
    }
  else
    data = {
      apikey: apikey,
      uri: start_response['uri'],
      region: start_response['region'],
      upload_id: start_response['upload_id'],
      filename: filename,
      size: filesize,
      mimetype: mimetype,
      store_location: options.nil? ? 's3' : options[:store_location],
      file: Tempfile.new(filename),
      'multipart' => 'true'
    }
  end
  data = data.merge!(options) if options

  Typhoeus.post(
    FilestackConfig::MULTIPART_COMPLETE_URL, body: data,
                                             headers: FilestackConfig::HEADERS
  )
end

#multipart_start(apikey, filename, filesize, mimetype, security, options) ⇒ Typhoeus::Response

Send start response to multipart endpoint

Parameters:

  • apikey (String) —

    Filestack API key

  • filename (String) —

    Name of incoming file

  • filesize (Int) —

    Size of incoming file

  • mimetype (String) —

    Mimetype of incoming file

  • security (FilestackSecurity) —

    Security object with policy/signature

  • options (Hash) —

    User-defined options for multipart uploads

Returns:

  • (Typhoeus::Response) 


37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 37

def multipart_start(apikey, filename, filesize, mimetype, security, options)
  params = {
    apikey: apikey,
    filename: filename,
    mimetype: mimetype,
    size: filesize,
    store_location: options.nil? ? 's3' : options[:store_location],
    file: Tempfile.new(filename),
    options: options,
    'multipart' => 'true'
  }
  params = params.merge!(options) if options

  unless security.nil?
    params[:policy] = security.policy
    params[:signature] = security.signature
  end

  response = Typhoeus.post(
    FilestackConfig::MULTIPART_START_URL, body: params,
                                          headers: FilestackConfig::HEADERS
  )
  if response.code == 200
    response.body
  else
    raise RuntimeError.new(response.body)
  end
end

#multipart_upload(apikey, filepath, security, options, timeout, intelligent: false) ⇒ Hash

Run entire multipart process through with file and options

Parameters:

  • apikey (String) —

    Filestack API key

  • filename (String) —

    Name of incoming file

  • security (FilestackSecurity) —

    Security object with policy/signature

  • options (Hash) —

    User-defined options for multipart uploads

Returns:



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 238

def multipart_upload(apikey, filepath, security, options, timeout, intelligent: false)
  filename, filesize, mimetype = get_file_info(filepath)
  start_response = multipart_start(
    apikey, filename, filesize, mimetype, security, options
  )
  unless start_response['upload_type'].nil?
    intelligent_enabled = ((start_response['upload_type'].include? 'intelligent_ingestion')) && intelligent
  end
  jobs = create_upload_jobs(
    apikey, filename, filepath, filesize, start_response, options
  )
  if intelligent_enabled
    state = IntelligentState.new
    run_intelligent_upload_flow(jobs, state)
    response_complete = multipart_complete(
      apikey, filename, filesize, mimetype,
      start_response, nil, options, intelligent
    )
  else
    parts_and_etags = run_uploads(jobs, apikey, filepath, options)
    response_complete = multipart_complete(
      apikey, filename, filesize, mimetype,
      start_response, parts_and_etags, options
    )
  end
  begin
    Timeout::timeout(timeout){
      while response_complete.code == 202
        response_complete = multipart_complete(
          apikey, filename, filesize, mimetype,
          start_response, nil, options, intelligent
        )
      end
    }
  rescue
    raise "Upload timed out upon completion. Please try again later"
  end
  JSON.parse(response_complete.body)
end

#run_uploads(jobs, apikey, filepath, options) ⇒ Array

Runs all jobs in parallel

Parameters:

  • jobs (Array) —

    Array of jobs to be run

  • apikey (String) —

    Filestack API key

  • filepath (String) —

    Local path to file

  • options (Hash) —

    User-defined options for multipart uploads

Returns:

  • (Array) —

    Array of parts/etags strings



160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 160

def run_uploads(jobs, apikey, filepath, options)
  bar = ProgressBar.new(jobs.length)
  results = Parallel.map(jobs, in_threads: 4) do |job|
    response = upload_chunk(
      job, apikey, filepath, options
    )
    if response.code == 200
      bar.increment!
      part = job[:part]
      etag = response.headers[:etag]
      "#{part}:#{etag}"
    end
  end
  results
end

#upload_chunk(job, apikey, filepath, options) ⇒ Typhoeus::Response

Uploads one chunk of the file

Parameters:

  • job (Hash) —

    Hash of options needed to upload a chunk

  • apikey (String) —

    Filestack API key

  • location_url (String) —

    Location url given back from endpoint

  • filepath (String) —

    Local path to file

  • options (Hash) —

    User-defined options for multipart uploads

Returns:

  • (Typhoeus::Response) 


124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/filestack/utils/multipart_upload_utils.rb', line 124

def upload_chunk(job, apikey, filepath, options)
  file = File.open(filepath)
  file.seek(job[:seek])
  chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)

  md5 = Digest::MD5.new
  md5 << chunk
  data = {
    apikey: apikey,
    part: job[:part],
    size: chunk.length,
    md5: md5.base64digest,
    uri: job[:uri],
    region: job[:region],
    upload_id: job[:upload_id],
    store_location: job[:store_location],
    file: Tempfile.new(job[:filename])
  }
  data = data.merge!(options) if options
  fs_response = Typhoeus.post(
    FilestackConfig::MULTIPART_UPLOAD_URL, body: data,
                                           headers: FilestackConfig::HEADERS
  ).body
  Typhoeus.put(
    fs_response['url'], headers: fs_response['headers'], body: chunk
  )
end