Class: BulkSubmission

Inherits:
Object
  • Object
show all
Extended by:
ActiveModel::Naming
Includes:
ActiveModel::AttributeMethods, ActiveModel::Conversion, ActiveModel::Validations, ManifestUtil, Submission::AssetSubmissionFinder
Defined in:
app/models/bulk_submission.rb

Overview

A bulk submission is created through the upload of a spreadsheet (csv) It contains the information for setting up one or more submissions, allowing for the quick request of multiple pieces of work simultaneously. Bulk Submissions are not currently persisted.

Constant Summary collapse

DEFAULT_ENCODING =

This is the default output from excel

'Windows-1252'
COMMON_FIELDS =

rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity

[
  # Needed to construct the submission ...
  'template name',
  'study id',
  'study name',
  'project id',
  'project name',
  'submission name',
  'user login',
  # Needed to identify the assets and what happens to them ...
  'asset group id',
  'asset group name',
  'fragment size from',
  'fragment size to',
  'pcr cycles',
  'primer panel',
  'read length',
  'library type',
  'bait library',
  'bait library name',
  'comments',
  'number of lanes',
  'pre-capture plex level',
  'pre-capture group',
  'gigabases expected',
  'priority'
].freeze
ALIAS_FIELDS = 
{ 'plate barcode' => 'barcode', 'tube barcode' => 'barcode' }.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from ManifestUtil

#filter_end_of_header, #is_end_of_header?

Methods included from Submission::AssetSubmissionFinder

#find_all_assets_by_name_including_samples!, #find_tubes_including_samples_for!, #find_wells_including_samples_for!, #is_plate?, #is_tube?

Constructor Details

#initialize(attrs = {}) ⇒ BulkSubmission

Returns a new instance of BulkSubmission.


51
52
53
54
 
# File 'app/models/bulk_submission.rb', line 51

def initialize(attrs = {})
  self.spreadsheet = attrs[:spreadsheet]
  self.encoding = attrs.fetch(:encoding, DEFAULT_ENCODING)
end

Instance Attribute Details

#encoding ⇒ Object

Returns the value of attribute encoding.


36
37
38
 
# File 'app/models/bulk_submission.rb', line 36

def encoding
  @encoding
end

#spreadsheet ⇒ Object

Returns the value of attribute spreadsheet.


36
37
38
 
# File 'app/models/bulk_submission.rb', line 36

def spreadsheet
  @spreadsheet
end

Instance Method Details

#add_study_to_assets(assets, study) ⇒ Object 


307
308
309
 
# File 'app/models/bulk_submission.rb', line 307

def add_study_to_assets(assets, study)
  assets.map(&:samples).flatten.uniq.each { |sample| sample.studies << study unless sample.studies.include?(study) }
end

#completed_submissions ⇒ Object

This is used to present a list of successes


442
443
444
 
# File 'app/models/bulk_submission.rb', line 442

def completed_submissions
  [@submission_ids, @completed_submissions]
end

#extract_request_options(details) ⇒ Object

rubocop:todo Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize


312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'app/models/bulk_submission.rb', line 312

def extract_request_options(details) # rubocop:todo Metrics/CyclomaticComplexity
  { read_length: details['read length'], multiplier: {} }.tap do |request_options|
    request_options['library_type'] = details['library type'] if details['library type'].present?
    if details['fragment size from'].present?
      request_options['fragment_size_required_from'] = details['fragment size from']
    end
    request_options['fragment_size_required_to'] = details['fragment size to'] if details['fragment size to'].present?
    request_options['pcr_cycles'] = details['pcr cycles'] if details['pcr cycles'].present?
    request_options[:bait_library_name] = details['bait library name'] if details['bait library name'].present?
    request_options[:bait_library_name] ||= details['bait library'] if details['bait library'].present?
    if details['pre-capture plex level'].present?
      request_options['pre_capture_plex_level'] = details['pre-capture plex level']
    end
    request_options['gigabases_expected'] = details['gigabases expected'] if details['gigabases expected'].present?
    request_options['primer_panel_name'] = details['primer panel'] if details['primer panel'].present?
  end
end

#find_template(template_name) ⇒ Object

Returns the SubmissionTemplate and checks that it is valid

Raises:

  • (StandardError) 

433
434
435
436
437
438
439
 
# File 'app/models/bulk_submission.rb', line 433

def find_template(template_name)
  template = SubmissionTemplate.find_by(name: template_name) or
    raise StandardError, "Cannot find template #{template_name}"
  raise(StandardError, "Template: '#{template_name}' is deprecated and no longer in use.") unless template.visible

  template
end

#id ⇒ Object 


47
48
49
 
# File 'app/models/bulk_submission.rb', line 47

def id
  nil
end

#persisted? ⇒ Boolean

Returns:

  • (Boolean) 

43
44
45
 
# File 'app/models/bulk_submission.rb', line 43

def persisted?
  false
end

#prepare_order(details) ⇒ Object

Returns an order for the given details rubocop:todo Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize


334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
 
# File 'app/models/bulk_submission.rb', line 334

def prepare_order(details) # rubocop:todo Metrics/CyclomaticComplexity
  # Retrieve common attributes
  study = Study.find_by_id_or_name!(details['study id'], details['study name'])
  project = Project.find_by_id_or_name!(details['project id'], details['project name'])
  user = User.find_by(login: details['user login']) or
    raise StandardError, "Cannot find user #{details['user login'].inspect}"

  # Extract the request options from the row details
  request_options = extract_request_options(details)

  # Check the library type matches a value from the table
  if request_options['library_type'].present?
    # find is case insensitive but we want the correct case sensitive name for requests or we get issues downstream in
    # NPG
    lt = LibraryType.find_by(name: request_options['library_type'])&.name or
      raise StandardError, "Cannot find library type #{request_options['library_type'].inspect}"
    request_options['library_type'] = lt
  end

  # Set up the order attributes
  attributes = {
    study: study,
    project: project,
    user: user,
    comments: details['comments'],
    request_options: request_options,
    pre_cap_group: details['pre-capture group']
  }

  # Deal with the asset group: either it's one we should be loading, or one we should be creating.

  attributes[:asset_group] =
    study.asset_groups.find_by_id_or_name(details['asset group id'], details['asset group name'])
  attributes[:asset_group_name] = details['asset group name'] if attributes[:asset_group].nil?

  ##
  # We go ahead and find our assets regardless of whether we have an asset group.
  # While this takes longer, it helps to detect cases where an asset group name has been
  # reused. This is a common cause of submission problems.

  # Locate either the assets by name or ID, or find the plate and it's well
  if is_plate?(details)
    found_assets = find_wells_including_samples_for!(details)
    # We've probably got a tube
  elsif is_tube?(details)
    found_assets = find_tubes_including_samples_for!(details)
  else
    asset_ids, asset_names = details.fetch('asset ids', ''), details.fetch('asset names', '')
    found_assets =
      if attributes[:asset_group] && asset_ids.blank? && asset_names.blank?
        []
      elsif asset_names.present?
        Array(find_all_assets_by_name_including_samples!(asset_names)).uniq
      elsif asset_ids.present?
        raise StandardError, 'Specifying assets by id is no longer possible. Please provide a name or barcode.'
      else
        raise StandardError, 'Please specify a barcode or name for each asset.'
      end

    assets_found, expecting =
      found_assets.map { |asset| "#{asset.name}(#{asset.id})" }, asset_ids.size + asset_names.size
    if assets_found.size < expecting
      raise StandardError, "Too few assets found for #{details['rows']}: #{assets_found.inspect}"
    end
    if assets_found.size > expecting
      raise StandardError, "Too many assets found for #{details['rows']}: #{assets_found.inspect}"
    end
  end

  if attributes[:asset_group].nil?
    attributes[:assets] = found_assets
  elsif found_assets.present? && found_assets != attributes[:asset_group].assets
    # rubocop:todo Layout/LineLength
    raise StandardError,
          "Asset Group '#{attributes[:asset_group].name}' contains different assets to those you specified. You may be reusing an asset group name"
    # rubocop:enable Layout/LineLength
  end

  add_study_to_assets(found_assets, study)

  # Create the order.  Ensure that the number of lanes is correctly set.
  sub_template = find_template(details['template name'])
  number_of_lanes = details.fetch('number of lanes', 1).to_i

  sub_template
    .new_order(attributes)
    .tap do |new_order|
      new_order.request_type_multiplier do |multiplexed_request_type_id|
        new_order.request_options[:multiplier][multiplexed_request_type_id] = number_of_lanes
      end
    end
rescue => e
  errors.add :spreadsheet, "There was a problem on row(s) #{details['rows']}: #{e.message}"
  nil
end

#process ⇒ Object

rubocop:todo Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize


142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
 
# File 'app/models/bulk_submission.rb', line 142

def process # rubocop:todo Metrics/CyclomaticComplexity
  # Store the details of the successful submissions so the user can be presented with a summary
  @submission_ids = []
  @completed_submissions = {}

  csv_content = spreadsheet.read
  @csv_rows = CSV.parse(csv_content.encode!('utf-8', encoding))

  if spreadsheet_valid?
    submission_details = submission_structure

    raise ActiveRecord::RecordInvalid, self if errors.count > 0

    # Within a single transaction process each of the rows of the CSV file as a separate submission.  Any name
    # fields need to be mapped to IDs, and the 'assets' field needs to be split up and processed if present.
    # rubocop:todo Metrics/BlockLength
    ActiveRecord::Base.transaction do
      submission_details.each do |submissions|
        submissions.each do |submission_name, orders|
          user = User.find_by(login: orders.first['user login'])
          if user.nil?
            errors.add(
              :spreadsheet,
              if orders.first['user login'].nil?
                "No user specified for #{submission_name}"
              else
                "Cannot find user #{orders.first['user login'].inspect}"
              end
            )
            next
          end

          begin
            orders_processed = orders.map(&method(:prepare_order)).compact

            submission =
              Submission.create!(
                name: submission_name,
                user: user,
                orders: orders_processed,
                priority: max_priority(orders)
              )
            submission.built!

            # Collect successful submissions
            @submission_ids << submission.id
            @completed_submissions[submission.id] =
              "Submission #{submission.id} built (#{submission.orders.count} orders)"
          rescue Submission::ProjectValidation::Error => e
            errors.add :spreadsheet, "There was an issue with a project: #{e.message}"
          end
        end
      end

      # If there are any errors then the transaction needs to be rolled back.
      raise ActiveRecord::Rollback if errors.present?
    end
    # rubocop:enable Metrics/BlockLength
  end
end

#process_file ⇒ Object

rubocop:todo Metrics/MethodLength


59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'app/models/bulk_submission.rb', line 59

def process_file # rubocop:todo Metrics/AbcSize
  # Slightly inelegant file-type checking
  # TODO (jr) Find a better way of verifying the CSV file?
  if spreadsheet.present?
    if spreadsheet.size == 0
      errors.add(:spreadsheet, 'The supplied file was empty')
    else
      if spreadsheet.original_filename.end_with?('.csv')
        process
      else
        errors.add(:spreadsheet, 'The supplied file was not a CSV file')
      end
    end
  end
rescue CSV::MalformedCSVError
  errors.add(:spreadsheet, 'The supplied file was not a valid CSV file (try opening it with MS Excel)')
rescue Encoding::InvalidByteSequenceError
  errors.add(:encoding, "didn't match for the provided file.")
end

#shared_options!(rows) ⇒ Object

rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity


290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
 
# File 'app/models/bulk_submission.rb', line 290

def shared_options!(rows) # rubocop:todo Metrics/MethodLength
  # Builds an array of the common fields. Raises and exception if the fields are inconsistent
  COMMON_FIELDS.map do |field|
    option = rows.map { |r| r[field] }.uniq
    if option.count > 1
      provided_values = option.map { |o| "'#{o}'" }.to_sentence
      errors.add(
        :spreadsheet,
        # rubocop:todo Layout/LineLength
        "#{field} should be identical for all requests in asset group '#{rows.first['asset group name']}'. Given values were: #{provided_values}."
        # rubocop:enable Layout/LineLength
      )
    end
    [field, option.first]
  end
end

#submission_structure ⇒ Object

Process CSV into a structure

this creates an array containing a hash for each distinct "submission name"
  "submission name" => array of orders
  where each order is a hash of headers to values (grouped by "asset group name")

rubocop:todo Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize


252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
 
# File 'app/models/bulk_submission.rb', line 252

def submission_structure # rubocop:todo Metrics/CyclomaticComplexity
  Hash
    .new { |h, i| h[i] = Array.new }
    .tap do |submission|
      csv_data_rows.each_with_index do |row, index|
        next if row.all?(&:nil?)

        details =
          headers
            .each_with_index
            .filter_map { |header, pos| validate_entry(header, pos, row, index + start_row) }
            .to_h
            .merge('row' => index + start_row)
        submission[details['submission name']] << details
      end
    end
    .map do |submission_name, rows|
      order =
        rows
          .group_by { |details| details['asset group name'] }
          .map do |_group_name, rows|
            shared_options!(rows)
              .to_h
              .tap do |details|
                details['rows'] = rows.comma_separate_field_list_for_display('row')
                details['asset ids'] = rows.field_list('asset id', 'asset ids')
                details['asset names'] = rows.field_list('asset name', 'asset names')
                details['plate well'] = rows.field_list('plate well')
                details['barcode'] = rows.field_list('barcode')
              end
              .delete_if { |_, v| v.blank? }
          end
      { submission_name => order }
    end
end

#translate(header) ⇒ Object 


235
236
237
 
# File 'app/models/bulk_submission.rb', line 235

def translate(header)
  ALIAS_FIELDS[header] || header
end

#valid_header? ⇒ Boolean

Returns:

  • (Boolean) 

115
116
117
118
119
120
121
122
123
124
125
 
# File 'app/models/bulk_submission.rb', line 115

def valid_header?
  return false if headers.nil?
  return true if headers.include? 'submission name'

  # rubocop:todo Layout/LineLength
  errors.add :spreadsheet,
             "You submitted an incompatible spreadsheet. Please ensure your spreadsheet contains the 'submission name' column"

  # rubocop:enable Layout/LineLength
  false
end