Class: NtqExcelsior::Importer

Inherits:
Object
  • Object
show all
Defined in:
lib/ntq_excelsior/importer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Importer

Returns a new instance of Importer.



64
65
66
 
# File 'lib/ntq_excelsior/importer.rb', line 64

def initialize
  @context = NtqExcelsior::Context.new
end

Instance Attribute Details

#check ⇒ Object

Returns the value of attribute check.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def check
  @check
end

#context ⇒ Object

Returns the value of attribute context.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def context
  @context
end

#file ⇒ Object

Returns the value of attribute file.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def file
  @file
end

#lines ⇒ Object

Returns the value of attribute lines.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def lines
  @lines
end

#options ⇒ Object

Returns the value of attribute options.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def options
  @options
end

#status_tracker ⇒ Object

Returns the value of attribute status_tracker.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def status_tracker
  @status_tracker
end

#success ⇒ Object

Returns the value of attribute success.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def success
  @success
end

Class Method Details

.after(&block) ⇒ Object 



38
39
40
41
 
# File 'lib/ntq_excelsior/importer.rb', line 38

def after(&block)
  @after = block if block_given?
  @after
end

.autosave(value = nil) ⇒ Object 



9
10
11
 
# File 'lib/ntq_excelsior/importer.rb', line 9

def autosave(value = nil)
  @autosave ||= value
end

.autoset(value = nil) ⇒ Object 



13
14
15
 
# File 'lib/ntq_excelsior/importer.rb', line 13

def autoset(value = nil)
  @autoset ||= value
end

.before(&block) ⇒ Object 



33
34
35
36
 
# File 'lib/ntq_excelsior/importer.rb', line 33

def before(&block)
  @before = block if block_given?
  @before
end

.description(value = nil) ⇒ Object 



59
60
61
 
# File 'lib/ntq_excelsior/importer.rb', line 59

def description(value = nil)
  @description ||= value
end

.max_error_count(value = nil) ⇒ Object 



43
44
45
 
# File 'lib/ntq_excelsior/importer.rb', line 43

def max_error_count(value = nil)
  @max_error_count ||= value
end

.model_klass(value = nil) ⇒ Object 



25
26
27
 
# File 'lib/ntq_excelsior/importer.rb', line 25

def model_klass(value = nil)
  @model_klass ||= value
end

.primary_key(value = nil) ⇒ Object 



21
22
23
 
# File 'lib/ntq_excelsior/importer.rb', line 21

def primary_key(value = nil)
  @primary_key ||= value
end

.sample_file(value = nil) ⇒ Object 



51
52
53
 
# File 'lib/ntq_excelsior/importer.rb', line 51

def sample_file(value = nil)
  @sample_file ||= value
end

.schema(value = nil) ⇒ Object 



29
30
31
 
# File 'lib/ntq_excelsior/importer.rb', line 29

def schema(value = nil)
  @schema ||= value
end

.spreadsheet_options(value = nil) ⇒ Object 



17
18
19
 
# File 'lib/ntq_excelsior/importer.rb', line 17

def spreadsheet_options(value = nil)
  @spreadsheet_options ||= value
end

.structure(value = nil) ⇒ Object 



47
48
49
 
# File 'lib/ntq_excelsior/importer.rb', line 47

def structure(value = nil)
  @structure ||= value
end

.title(value = nil) ⇒ Object 



55
56
57
 
# File 'lib/ntq_excelsior/importer.rb', line 55

def title(value = nil)
  @title ||= value
end

Instance Method Details

#detect_header_scheme ⇒ Object

Detect header scheme This method will detect the header scheme based on the schema

return [Hash] header_scheme - Array<{ [spreadsheet_header]: schema_key }>



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/ntq_excelsior/importer.rb', line 125

def detect_header_scheme
  return @header_scheme if defined?(@header_scheme)

  @header_scheme = {}
  # Read the first line of file (not header)
  l = spreadsheet_data[0].dup || []

  self.class.schema.each do |field, column_config|
    header = column_config.is_a?(Hash) ? column_config[:header] : column_config
    l.each do |parsed_header, _value|
      next unless parsed_header

      unless header.is_a?(String) && parsed_header == header || (header.is_a?(Regexp) && parsed_header.respond_to?(:match?) && parsed_header.match?(header))
        next
      end

      l.delete(parsed_header)
      @header_scheme[parsed_header] = field
    end
  end
  if self.class.primary_key && !self.class.schema[self.class.primary_key.to_sym]
    @header_scheme[self.class.primary_key.to_s] =
      self.class.primary_key.to_s
  end
  @header_scheme
end

#find_or_initialize_record(line) ⇒ Object

id for default query in model line in case an override is needed to find correct record



186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/ntq_excelsior/importer.rb', line 186

def find_or_initialize_record(line)
  return nil unless self.class.primary_key && self.class.model_klass

  if line[self.class.primary_key.to_sym]
    if self.class.primary_key.to_sym == :id
      record = self.class.model_klass.constantize.find_by id: line[self.class.primary_key.to_sym]
    else
      record = self.class.model_klass.constantize.find_or_initialize_by("#{self.class.primary_key}": line[self.class.primary_key.to_sym])
    end
  end
  record ||= self.class.model_klass.constantize.new
  record
end

#import(save: true, status_tracker: nil) ⇒ Object 



241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'lib/ntq_excelsior/importer.rb', line 241

def import(save: true, status_tracker: nil)
  self.class.before.call(@context, options) if self.class.before.is_a?(Proc)
  at = 0
  errors_lines = []
  success_count = 0
  not_found_count = 0
  lines.each_with_index do |line, index|
    break if errors_lines.size == self.class.max_error_count

    result = import_line(line.with_indifferent_access, save: true)
    case result[:status]
    when :not_found
      not_found_count += 1
    when :success
      success_count += 1
    when :error
      error_line = line.map { |_k, v| v }
      error_line << result[:errors]
      errors_lines.push(error_line)
    end

    if @status_tracker&.is_a?(Proc)
      at = (((index + 1).to_d / lines.size) * 100.to_d)
      @status_tracker.call(at)
    end
  end

  import_stats = { success_count: success_count, not_found_count: not_found_count, errors: errors_lines }
  @context.success = true if errors_lines.empty?
  self.class.after.call(@context, options) if self.class.after.is_a?(Proc)
  import_stats
end

#import_line(line, save: true) {|record, line| ... } ⇒ Object

Yields:

  • (record, line) 


214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
 
# File 'lib/ntq_excelsior/importer.rb', line 214

def import_line(line, save: true)
  record = find_or_initialize_record(line)
  return { status: :not_found } unless record

  @success = false
  @action = nil
  @errors = []

  record = set_record_fields(record, line) if self.class.autoset

  yield(record, line) if block_given?

  if self.class.autosave.nil? || self.class.autosave
    @action = record.persisted? ? "update" : "create"
    @success = if save
                 record.save
               else
                 record.valid?
               end
    @errors = record.errors.full_messages.concat(@errors) if record.errors.any?
  end

  return { status: :success, action: @action } if @success

  { status: :error, errors: @errors.join(", ") }
end

#parse_line(line) ⇒ Object 



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/ntq_excelsior/importer.rb', line 156

def parse_line(line)
  parsed_line = {}
  header_scheme = detect_header_scheme
  line.each do |header, value|
    if header.to_s == self.class.primary_key.to_s
      parsed_line[self.class.primary_key] = value
      next
    end
  end

  header_scheme.each do |spreadsheet_header, schema_key|
    header_config = schema_config_for_key(schema_key)
    parsed_value = line[spreadsheet_header]
    if header_config&.is_a?(Hash) && header_config[:parser].is_a?(Proc)
      parsed_value = header_config[:parser].call(parsed_value)
    end
    parsed_line[schema_key.to_sym] = parsed_value
  end

  parsed_line
end

#record_attributes(record) ⇒ Object 



200
201
202
203
204
 
# File 'lib/ntq_excelsior/importer.rb', line 200

def record_attributes(record)
  return @record_attributes if @record_attributes

  @record_attributes = self.class.schema.keys.select { |k| k.to_sym != :id && record.respond_to?(:"#{k}=") }
end

#required_headers ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/ntq_excelsior/importer.rb', line 76

def required_headers
  return @required_headers if @required_headers

  @required_columns = self.class.schema.select do |_field, column_config|
    !column_config.is_a?(Hash) || !column_config.key?(:required) || column_config[:required]
  end
  @required_headers = @required_columns.values.map do |column|
                        get_column_header(column)
                      end.map { |header| transform_header_to_regexp(header) }
  if self.class.primary_key && !@required_columns.keys.include?(self.class.primary_key)
    @required_headers.unshift(Regexp.new(self.class.primary_key.to_s, "i"))
  end
  @required_headers
end

#schema_config_for_key(key) ⇒ Object 



152
153
154
 
# File 'lib/ntq_excelsior/importer.rb', line 152

def schema_config_for_key(key)
  self.class.schema[key.to_sym]
end

#set_record_fields(record, line) ⇒ Object 



206
207
208
209
210
211
212
 
# File 'lib/ntq_excelsior/importer.rb', line 206

def set_record_fields(record, line)
  attributes_to_set = record_attributes(record)
  attributes_to_set.each do |attribute|
    record.send(:"#{attribute}=", line[attribute])
  end
  record
end

#spreadsheet ⇒ Object 



68
69
70
71
72
73
74
 
# File 'lib/ntq_excelsior/importer.rb', line 68

def spreadsheet
  return @spreadsheet unless @spreadsheet.nil?

  raise "File is missing" unless file

  @spreadsheet = Roo::Spreadsheet.open(file, self.class.spreadsheet_options || {})
end

#spreadsheet_data ⇒ Object 



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/ntq_excelsior/importer.rb', line 91

def spreadsheet_data
  spreadsheet_data = spreadsheet.sheet(spreadsheet.sheets[0]).parse(header_search: required_headers)
  unless spreadsheet_data.size > 0
    raise "File is inconsistent, please check you have data in it or check for invalid characters in headers like , / ; etc..."
  end

  spreadsheet_data
rescue Roo::HeaderRowNotFoundError => e
  missing_headers = []

  e.message.delete_prefix("[").delete_suffix("]").split(",").map(&:strip).each do |header_missing|
    header_missing_regex = transform_header_to_regexp(header_missing, true)
    header_found = @required_columns.values.find do |column|
      transform_header_to_regexp(get_column_header(column)) == header_missing_regex
    end
    missing_headers << if header_found && header_found.is_a?(Hash)
                         if header_found[:header].is_a?(String)
                           header_found[:header]
                         else
                           (header_found[:humanized_header] || header_missing)
                         end
                       elsif header_found&.is_a?(String)
                         header_found
                       else
                         header_missing
                       end
  end
  raise Roo::HeaderRowNotFoundError, missing_headers.join(", ")
end