Class: Flatfish::Pleuronectiformes

Inherits:
Object
  • Object
show all
Defined in:
lib/flatfish/pleuronectiformes.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(ymal) ⇒ Pleuronectiformes

load in the config



9
10
11
12
13
 
# File 'lib/flatfish/pleuronectiformes.rb', line 9

def initialize(ymal)
  @config = YAML.load_file(ymal)
  db_conn() # establish AR conn
  @klasses = Hash.new
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



6
7
8
 
# File 'lib/flatfish/pleuronectiformes.rb', line 6

def config
  @config
end

#klasses ⇒ Object (readonly)

Returns the value of attribute klasses.



6
7
8
 
# File 'lib/flatfish/pleuronectiformes.rb', line 6

def klasses
  @klasses
end

#schema ⇒ Object (readonly)

Returns the value of attribute schema.



6
7
8
 
# File 'lib/flatfish/pleuronectiformes.rb', line 6

def schema
  @schema
end

Instance Method Details

#create_klass(k) ⇒ Object

Create the Klass create table if necessary: table must exist! create dynamic model



32
33
34
35
36
37
38
 
# File 'lib/flatfish/pleuronectiformes.rb', line 32

def create_klass(k)
    # commence hackery
    create_table(k) unless ActiveRecord::Base.connection.tables.include?(k.tableize)
    @klass = Class.new(Page)
    @klasses[k] = @klass
    @klass.table_name = k.tableize
end

#create_media ⇒ Object 



45
46
47
 
# File 'lib/flatfish/pleuronectiformes.rb', line 45

def create_media
  Flatfish::CreateMedia.setup
end

#create_table(klass) ⇒ Object 



40
41
42
43
 
# File 'lib/flatfish/pleuronectiformes.rb', line 40

def create_table(klass)
  load_csv
  Flatfish::CreateKlass.setup(@schema, klass)
end

#db_conn ⇒ Object 



111
112
113
114
115
116
117
118
119
 
# File 'lib/flatfish/pleuronectiformes.rb', line 111

def db_conn
  ActiveRecord::Base.establish_connection(
                                    :adapter=> "mysql2",
                                    :host => "localhost",
                                    :username => @config['db_user'],
                                    :password => @config['db_pass'],
                                    :database=> @config['db']
                                   )
end

#ingurgitate ⇒ Object

main loop for flatfish



16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/flatfish/pleuronectiformes.rb', line 16

def ingurgitate
  create_media unless Flatfish::Media.table_exists?

  @config["types"].each do |k,v|
    next if v["csv"].nil?
    @csv_file = v["csv"]
    @host = v["host"]
    create_klass(k)
    parse(k)
  end
  output_schema
end

#load_csv ⇒ Object

load csv, set schema



50
51
52
53
54
 
# File 'lib/flatfish/pleuronectiformes.rb', line 50

def load_csv
  csv = CSV.read(@csv_file)
  @schema = csv.shift[3..-1]
  return csv
end

#output_schema ⇒ Object

generate a dynamic schema.yml for Migrate mapping



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/flatfish/pleuronectiformes.rb', line 81

def output_schema
  # TODO REFACTOR THIS ISH
  klasses = @klasses
  File.open('schema.yml', 'w') do |out|
    output = Hash.new
    output["schema"] = Hash.new
    klasses["Media"] = Flatfish::Media

    klasses.each_pair do |k,v|
      klass_attributes = Hash.new
      v.new.attributes.each { |a| klass_attributes[a[0]] = split_type(v.columns_hash[a[0]].sql_type) }
      output["schema"].merge!({k => {"machine_name" => k.tableize, "fields" => klass_attributes, "primary key" => ["id"]}})
    end
    out.write output.to_yaml
  end
end

#parse(k) ⇒ Object

loop thru csv



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/flatfish/pleuronectiformes.rb', line 57

def parse(k)
  csv = load_csv
  @cnt = 0
  csv.each do |row|
    begin
      break if @cnt == @config['max_rows']
      @cnt += 1
      page = @klass.find_or_create_by_url(row[0])
      puts "Processing #{k}.#{page.id} with URL #{row[0]}"
      page.setup(row, @config, @schema, @host)
      page.process
      page.save!
    rescue Exception => e
      if e.message =~ /(redirection forbidden|404 Not Found)/
        ap "URL: #{page.url} #{e}"
      else
        ap "URL: #{page.url} ERROR: #{e} BACKTRACE: #{e.backtrace}"
      end
    end
  end
end

#split_type(type) ⇒ Object

helper function to convert AR sql_type to Drupal format; eg :type => varchar(255) to :type => varchar, :length => 255



101
102
103
104
105
106
107
108
 
# File 'lib/flatfish/pleuronectiformes.rb', line 101

def split_type type
  if type =~ /\(/ then
    x = type.split("(")
    return {"type" => x[0], "length" => x[1].sub(")","").to_i }
  else
    return {"type" => type}
  end
end