Module: Importable
- Extended by:
- ActiveSupport::Concern
- Included in:
- Importo::BaseImporter
- Defined in:
- app/importers/concerns/importable.rb
Instance Method Summary collapse
- #after_build(_record, _row) ⇒ Object
- #after_save(_record, _row) ⇒ Object
- #after_validate(_record, _row) ⇒ Object
- #around_build(_record, _row) ⇒ Object
- #around_save(_record, _row) ⇒ Object
- #around_validate(_record, _row) ⇒ Object
-
#before_build(_record, _row) ⇒ Object
Callbakcs.
- #before_save(_record, _row) ⇒ Object
- #before_validate(_record, _row) ⇒ Object
-
#build(row) ⇒ Object
Build a record based on the row, when you override build, depending on your needs you will need to call populate yourself, or skip this altogether.
- #convert_values(row) ⇒ Object
-
#import! ⇒ Object
Does the actual import.
-
#populate(row, record = nil) ⇒ Object
Assists in pre-populating the record for you It wil try and find the record by id, or initialize a new record with it’s attributes set based on the mapping from columns.
- #process_data_row(attributes, index, last_attempt: true) ⇒ Object
Instance Method Details
#after_build(_record, _row) ⇒ Object
100 101 |
# File 'app/importers/concerns/importable.rb', line 100 def after_build(_record, _row) end |
#after_save(_record, _row) ⇒ Object
110 111 |
# File 'app/importers/concerns/importable.rb', line 110 def after_save(_record, _row) end |
#after_validate(_record, _row) ⇒ Object
120 121 |
# File 'app/importers/concerns/importable.rb', line 120 def after_validate(_record, _row) end |
#around_build(_record, _row) ⇒ Object
96 97 98 |
# File 'app/importers/concerns/importable.rb', line 96 def around_build(_record, _row) yield end |
#around_save(_record, _row) ⇒ Object
106 107 108 |
# File 'app/importers/concerns/importable.rb', line 106 def around_save(_record, _row) yield end |
#around_validate(_record, _row) ⇒ Object
116 117 118 |
# File 'app/importers/concerns/importable.rb', line 116 def around_validate(_record, _row) yield end |
#before_build(_record, _row) ⇒ Object
Callbakcs
93 94 |
# File 'app/importers/concerns/importable.rb', line 93 def before_build(_record, _row) end |
#before_save(_record, _row) ⇒ Object
103 104 |
# File 'app/importers/concerns/importable.rb', line 103 def before_save(_record, _row) end |
#before_validate(_record, _row) ⇒ Object
113 114 |
# File 'app/importers/concerns/importable.rb', line 113 def before_validate(_record, _row) end |
#build(row) ⇒ Object
Build a record based on the row, when you override build, depending on your needs you will need to call populate yourself, or skip this altogether.
12 13 14 |
# File 'app/importers/concerns/importable.rb', line 12 def build(row) populate(row) end |
#convert_values(row) ⇒ Object
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
# File 'app/importers/concerns/importable.rb', line 16 def convert_values(row) return row if row.instance_variable_get(:@importo_converted_values) row.instance_variable_set(:@importo_converted_values, true) columns.each do |k, col| next if col.proc.blank? || row[k].nil? attr = col.[:attribute] row[k] = import.column_overrides[col.attribute] if import.column_overrides[col.attribute] if col.collection # see if the value is part of the collection of (name, id) pairs, error if not. value = col.collection.find { |item| item.last == row[k] || item.first == row[k] }&.last raise StandardError, "#{row[k]} is not a valid value for #{col.name}" if value.nil? && row[k].present? else value ||= row[k] end if value.present? && col.proc proc = col.proc proc_result = instance_exec value, row, &proc value = proc_result if proc_result end value ||= col.[:default] row[k] = value end row end |
#import! ⇒ Object
Does the actual import
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
# File 'app/importers/concerns/importable.rb', line 126 def import! raise ArgumentError, "Invalid data structure" unless structure_valid? batch = Sidekiq::Batch.new batch.description = "#{import.original.filename} - #{import.kind}" batch.on(:success, Importo::ImportJobCallback, import_id: import.id) batch.jobs do column_with_delay = columns.select { |k, v| v.delay.present? } loop_data_rows do |attributes, index| if column_with_delay.present? delay = column_with_delay.map do |k, v| next unless attributes[k].present? v.delay.call(attributes[k]) end.compact end Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_async(JSON.dump(attributes), index, import.id) if delay.present? Importo::ImportJob.perform_async(JSON.dump(attributes), index, import.id) unless delay.present? end end true rescue => e @import. = "Exception: #{e.}" Rails.logger.error "Importo exception: #{e.} backtrace #{e.backtrace.join(";")}" @import.failure! false end |
#populate(row, record = nil) ⇒ Object
Assists in pre-populating the record for you It wil try and find the record by id, or initialize a new record with it’s attributes set based on the mapping from columns
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
# File 'app/importers/concerns/importable.rb', line 52 def populate(row, record = nil) raise "No attributes set for columns" unless columns.any? { |_, v| v.[:attribute].present? } row = convert_values(row) result = if record record else raise "No model set" unless model model.find_or_initialize_by(id: row["id"]) end attributes = {} cols_to_populate = columns.select do |_, v| v.[:attribute].present? end cols_to_populate = cols_to_populate.deep_symbolize_keys cols_to_populate.each do |k, col| attr = col.[:attribute] next unless row.key? k next if !row[k].present? && col.[:default].nil? attributes = if !row[k].present? && !col.[:default].nil? set_attribute(attributes, attr, col.[:default]) else set_attribute(attributes, attr, row[k]) end end result.assign_attributes(attributes) result end |
#process_data_row(attributes, index, last_attempt: true) ⇒ Object
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 |
# File 'app/importers/concerns/importable.rb', line 156 def process_data_row(attributes, index, last_attempt: true) record = nil row_hash = Digest::SHA256.base64digest(attributes.inspect) duplicate_import = nil run_callbacks :row_import do record = nil ActiveRecord::Base.transaction do register_result(index, hash: row_hash, state: :processing) before_build(record, attributes) around_build(record, attributes) do record = build(attributes) end after_build(record, attributes) before_validate(record, attributes) around_validate(record, attributes) do record.validate! end after_validate(record, attributes) model.with_advisory_lock(:importo) do before_save(record, attributes) around_save(record, attributes) do record.save! end after_save(record, attributes) end duplicate_import = duplicate?(row_hash, record.id) raise Importo::DuplicateRowError if duplicate_import register_result(index, class: record.class.name, id: record.id, state: :success) end end record rescue Importo::DuplicateRowError record_id = duplicate_import.results.find { |data| data["hash"] == row_hash }["id"] register_result(index, id: record_id, state: :duplicate, message: "Row already imported successfully on #{duplicate_import.created_at.to_date}") run_callbacks(:row_import, :after) nil rescue Importo::RetryError => e raise e unless last_attempt errors = record.respond_to?(:errors) && record.errors..join(", ") = "#{e.} (#{e.backtrace.first.split("/").last})" failure(attributes, record, index, e) register_result(index, class: record.class.name, state: :failure, message: , errors: errors) nil rescue => e # We rescue ActiveRecord::RecordNotUnique here, due to how transactions work, some row imports may have started the transaction and the current row doesn't see the results raise Importo::RetryError.new("ActiveRecord::RecordNotUnique", 2) if !last_attempt && e.is_a?(ActiveRecord::RecordNotUnique) errors = record.respond_to?(:errors) && record.errors..join(", ") = "#{e.} (#{e.backtrace.first.split("/").last})" failure(attributes, record, index, e) register_result(index, class: record.class.name, state: :failure, message: , errors: errors) run_callbacks(:row_import, :after) nil end |