Class: Ferry::Importer

Inherits:
Utilities show all
Defined in:
lib/ferry/importer.rb

Overview

TODO things to consider for importer

  • columns not matching and handling those errors

Instance Method Summary collapse

Methods inherited from Utilities

#check_valid_db, #check_valid_filetype, #db_connect, #execute, #print_version

Instance Method Details

#import_csv(environment, model, filename) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/ferry/importer.rb', line 32

def import_csv(environment, model, filename)
  if(File.extname(filename) != ".csv")
    raise "Please choose a csv file"
    return false
  end
  db_connect(environment)
  adapter = YAML::load(IO.read("config/database.yml"))[environment]["adapter"]
  lines = CSV.read(filename)
  if(lines.nil?)
    raise "#{filename} file not found"
    return false
  end
  import_bar = ProgressBar.new("import", lines.length-1)
  col_names = lines.shift
  records = []
  lines.each do |line|
    record = Hash[col_names.zip line]
    records << record
  end
  values = []
  records.map do |record|
    values << row_sql_format(record, col_names, adapter)
    import_bar.inc
  end
  insert_sql(model, col_names, values)
  puts ""
  puts "csv imported to #{model} table"
end

#import_json(environment, model, filename) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/ferry/importer.rb', line 61

def import_json(environment, model, filename)
  if(File.extname(filename) != ".json")
    raise "Please choose a json file"
    return false
  end
  db_connect(environment)
  adapter = YAML::load(IO.read("config/database.yml"))[environment]["adapter"]
  file = File.read(filename)
  column_titles = JSON.parse(file).first.keys
  data = JSON.parse(file)
  import_bar = ProgressBar.new("import", data.length-1)
  records = []
  data.each do |record|
    record = Hash[record]
    records << record
  end
  values = []
  records.map do |record|
    values << row_sql_format(record, column_titles, adapter)
    import_bar.inc
  end
  insert_sql(model, column_titles, values)
  puts ""
  puts "json imported to #{model} table"
end

#insert_sql(model, columns, values) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/ferry/importer.rb', line 20

def insert_sql(model, columns, values)
  col_names_sql = "(#{columns.join(",")})"
  model_sql = model.downcase
  sql_insert_beg = "INSERT INTO #{model_sql} #{col_names_sql} VALUES "
  ActiveRecord::Base.transaction do
    values.each_slice(500) do |records|
      sql_statement = sql_insert_beg + records.join(",") + ";"
      ActiveRecord::Base.connection.execute(sql_statement)
    end
  end
end

#row_sql_format(hash, columns, adapter) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
 
# File 'lib/ferry/importer.rb', line 9

def row_sql_format(hash, columns, adapter)
  values = hash.values_at(*columns)
  values.map! do |value|
    if(adapter=="mysql2" && (value=='t' || value =='f'))
      value=='t' ? value=1 : value=0
    end
    value = ActiveRecord::Base::sanitize(value)
  end
  "(#{values.join(",")})"
end