Class: Bulkrax::Importer

Inherits:
ApplicationRecord show all
Includes:
ImporterExporterBehavior, StatusInfo
Defined in:
app/models/bulkrax/importer.rb

Constant Summary collapse

DEFAULT_OBJECT_TYPES = 
%w[collection work file_set relationship].freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from StatusInfo

#current_status, #failed?, #last_error, #set_status_info, #status_at, #succeeded?

Methods included from ImporterExporterBehavior

#file?, #increment_counters, #key_without_numbers, #keys_without_numbers, #last_imported_at, #next_import_at, #parser, #parser_class, #zip?

Instance Attribute Details

#current_run ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'app/models/bulkrax/importer.rb', line 106

def current_run
  return @current_run if @current_run.present?

  @current_run = self.importer_runs.create!
  return @current_run if file? && zip?

  entry_counts = {
    total_work_entries: self.limit || parser.works_total,
    total_collection_entries: parser.collections_total,
    total_file_set_entries: parser.file_sets_total
  }
  @current_run.update!(entry_counts)

  @current_run
end

#file ⇒ Object

Returns the value of attribute file.



23
24
25
 
# File 'app/models/bulkrax/importer.rb', line 23

def file
  @file
end

#file_style ⇒ Object

Returns the value of attribute file_style.



23
24
25
 
# File 'app/models/bulkrax/importer.rb', line 23

def file_style
  @file_style
end

#only_updates ⇒ Object

Returns the value of attribute only_updates.



23
24
25
 
# File 'app/models/bulkrax/importer.rb', line 23

def only_updates
  @only_updates
end

Class Method Details

.frequency_enums ⇒ Object 



86
87
88
89
90
91
 
# File 'app/models/bulkrax/importer.rb', line 86

def self.frequency_enums
  # these duration values use ISO 8601 Durations (https://en.wikipedia.org/wiki/ISO_8601#Durations)
  # TLDR; all durations are prefixed with 'P' and the parts are a number with the type of duration.
  # i.e. P1Y2M3W4DT5H6M7S == 1 Year, 2 Months, 3 Weeks, 4 Days, 5 Hours, 6 Minutes, 7 Seconds
  [['Daily', 'P1D'], ['Monthly', 'P1M'], ['Yearly', 'P1Y'], ['Once (on save)', 'PT0S']]
end

.safe_uri_filename(uri) ⇒ Object 



26
27
28
29
30
31
32
33
 
# File 'app/models/bulkrax/importer.rb', line 26

def self.safe_uri_filename(uri)
  r = Faraday.head(uri.to_s)
  return CGI.parse(r.headers['content-disposition'])["filename"][0].delete("\"")
rescue
  filename = File.basename(uri.to_s)
  filename.delete!('/')
  filename.presence || SecureRandom.uuid
end

Instance Method Details

#default_field_mapping ⇒ Object 



72
73
74
75
76
77
78
79
80
 
# File 'app/models/bulkrax/importer.rb', line 72

def default_field_mapping
  return self.field_mapping if parser.import_fields.nil?

  ActiveSupport::HashWithIndifferentAccess.new(
    parser.import_fields.reject(&:nil?).map do |m|
      Bulkrax.default_field_mapping.call(m)
    end.inject(:merge)
  )
end

#errored_entries_csv_path ⇒ Object 



186
187
188
 
# File 'app/models/bulkrax/importer.rb', line 186

def errored_entries_csv_path
  @errored_entries_csv_path ||= File.join(parser.base_path, "import_#{path_string}_errored_entries.csv")
end

#frequency ⇒ Object 



97
98
99
100
 
# File 'app/models/bulkrax/importer.rb', line 97

def frequency
  f = self[:frequency] || "PT0S"
  ISO8601::Duration.new(f)
end

#frequency=(frequency) ⇒ Object 



93
94
95
 
# File 'app/models/bulkrax/importer.rb', line 93

def frequency=(frequency)
  self[:frequency] = ISO8601::Duration.new(frequency).to_s
end

#import_collections ⇒ Object 



142
143
144
 
# File 'app/models/bulkrax/importer.rb', line 142

def import_collections
  import_objects(['collection'])
end

#import_file_sets ⇒ Object 



146
147
148
 
# File 'app/models/bulkrax/importer.rb', line 146

def import_file_sets
  import_objects(['file_set'])
end

#import_metadata_format ⇒ Object

The format for metadata for the incoming import; corresponds to an Entry class



172
173
174
 
# File 'app/models/bulkrax/importer.rb', line 172

def import_metadata_format
  [['CSV', 'Bulkrax::CsvEntry'], ['RDF (N-Triples)', 'Bulkrax::RdfEntry']]
end

#import_objects(types_array = nil) ⇒ Object 



156
157
158
159
160
161
162
163
 
# File 'app/models/bulkrax/importer.rb', line 156

def import_objects(types_array = nil)
  self.only_updates ||= false
  self.save if self.new_record? # Object needs to be saved for statuses
  types = types_array || DEFAULT_OBJECT_TYPES
  parser.create_objects(types)
rescue StandardError => e
  set_status_info(e)
end

#import_relationships ⇒ Object 



150
151
152
 
# File 'app/models/bulkrax/importer.rb', line 150

def import_relationships
  import_objects(['relationship'])
end

#import_works ⇒ Object 



138
139
140
 
# File 'app/models/bulkrax/importer.rb', line 138

def import_works
  import_objects(['work'])
end

#importer_unzip_path ⇒ Object

If the import data is zipped, unzip it to this path



182
183
184
 
# File 'app/models/bulkrax/importer.rb', line 182

def importer_unzip_path
  @importer_unzip_path ||= File.join(parser.base_path, "import_#{path_string}")
end

#last_run ⇒ Object 



122
123
124
 
# File 'app/models/bulkrax/importer.rb', line 122

def last_run
  @last_run ||= self.importer_runs.last
end

#mapping ⇒ Object

If field_mapping is empty, setup a default based on the export_properties



59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'app/models/bulkrax/importer.rb', line 59

def mapping
  # rubocop:disable Style/IfUnlessModifier
  @mapping ||= if self.field_mapping.blank? || self.field_mapping == [{}]
                 if parser.import_fields.present? || self.field_mapping == [{}]
                   default_field_mapping
                 end
               else
                 default_field_mapping.merge(self.field_mapping)
               end

  # rubocop:enable Style/IfUnlessModifier
end

#metadata_only? ⇒ Boolean

Returns:

  • (Boolean) 


196
197
198
 
# File 'app/models/bulkrax/importer.rb', line 196

def metadata_only?
  parser.parser_fields['metadata_only'] == true
end

#parser_fields ⇒ Object 



82
83
84
 
# File 'app/models/bulkrax/importer.rb', line 82

def parser_fields
  self[:parser_fields] || {}
end

#path_string ⇒ Object 



190
191
192
193
194
 
# File 'app/models/bulkrax/importer.rb', line 190

def path_string
  "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}_#{self.importer_runs.last.id}"
rescue
  "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
end

#record_status ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'app/models/bulkrax/importer.rb', line 43

def record_status
  importer_run = ImporterRun.find(current_run.id) # make sure fresh
  return if importer_run.enqueued_records.positive? # still processing
  if importer_run.failed_records.positive?
    if importer_run.invalid_records.present?
      e = Bulkrax::ImportFailed.new('Failed with Invalid Records', importer_run.invalid_records.split("\n"))
      importer_run.importer.set_status_info(e)
    else
      importer_run.importer.set_status_info('Complete (with failures)')
    end
  else
    importer_run.importer.set_status_info('Complete')
  end
end

#replace_files ⇒ Object 



130
131
132
 
# File 'app/models/bulkrax/importer.rb', line 130

def replace_files
  self.parser_fields['replace_files']
end

#schedulable? ⇒ Boolean

Returns:

  • (Boolean) 


102
103
104
 
# File 'app/models/bulkrax/importer.rb', line 102

def schedulable?
  frequency.to_seconds != 0
end

#seen ⇒ Object 



126
127
128
 
# File 'app/models/bulkrax/importer.rb', line 126

def seen
  @seen ||= {}
end

#status ⇒ Object 



35
36
37
38
39
40
41
 
# File 'app/models/bulkrax/importer.rb', line 35

def status
  if self.validate_only
    'Validated'
  else
    super
  end
end

#unique_collection_identifier(id) ⇒ Object

TODO:

  • move to parser, as this is OAI specific

Prepend the base_url to ensure unique set identifiers



167
168
169
 
# File 'app/models/bulkrax/importer.rb', line 167

def unique_collection_identifier(id)
  "#{self.parser_fields['base_url'].split('/')[2]}_#{id}"
end

#update_files ⇒ Object 



134
135
136
 
# File 'app/models/bulkrax/importer.rb', line 134

def update_files
  self.parser_fields['update_files']
end