Class: Flatfish::Pleuronectiformes
- Inherits:
-
Object
- Object
- Flatfish::Pleuronectiformes
- Defined in:
- lib/flatfish/pleuronectiformes.rb
Instance Attribute Summary collapse
-
#config ⇒ Object
readonly
Returns the value of attribute config.
-
#klasses ⇒ Object
readonly
Returns the value of attribute klasses.
-
#schema ⇒ Object
readonly
Returns the value of attribute schema.
Instance Method Summary collapse
-
#create_klass(k) ⇒ Object
Create the Klass create table if necessary: table must exist! create dynamic model.
- #create_media ⇒ Object
- #create_table(klass) ⇒ Object
- #db_conn ⇒ Object
-
#ingurgitate ⇒ Object
main loop for flatfish.
-
#initialize(ymal) ⇒ Pleuronectiformes
constructor
load in the config.
-
#load_csv ⇒ Object
load csv, set schema.
-
#output_schema ⇒ Object
generate a dynamic schema.yml for Migrate mapping.
-
#parse(k) ⇒ Object
loop thru csv.
-
#split_type(type) ⇒ Object
helper function to convert AR sql_type to Drupal format; eg :type => varchar(255) to :type => varchar, :length => 255.
Constructor Details
#initialize(ymal) ⇒ Pleuronectiformes
load in the config
9 10 11 12 13 |
# File 'lib/flatfish/pleuronectiformes.rb', line 9 def initialize(ymal) @config = YAML.load_file(ymal) db_conn() # establish AR conn @klasses = Hash.new end |
Instance Attribute Details
#config ⇒ Object (readonly)
Returns the value of attribute config.
6 7 8 |
# File 'lib/flatfish/pleuronectiformes.rb', line 6 def config @config end |
#klasses ⇒ Object (readonly)
Returns the value of attribute klasses.
6 7 8 |
# File 'lib/flatfish/pleuronectiformes.rb', line 6 def klasses @klasses end |
#schema ⇒ Object (readonly)
Returns the value of attribute schema.
6 7 8 |
# File 'lib/flatfish/pleuronectiformes.rb', line 6 def schema @schema end |
Instance Method Details
#create_klass(k) ⇒ Object
Create the Klass create table if necessary: table must exist! create dynamic model
32 33 34 35 36 37 38 |
# File 'lib/flatfish/pleuronectiformes.rb', line 32 def create_klass(k) # commence hackery create_table(k) unless ActiveRecord::Base.connection.tables.include?(k.tableize) @klass = Class.new(Page) @klasses[k] = @klass @klass.table_name = k.tableize end |
#create_media ⇒ Object
45 46 47 |
# File 'lib/flatfish/pleuronectiformes.rb', line 45 def create_media Flatfish::CreateMedia.setup end |
#create_table(klass) ⇒ Object
40 41 42 43 |
# File 'lib/flatfish/pleuronectiformes.rb', line 40 def create_table(klass) load_csv Flatfish::CreateKlass.setup(@schema, klass) end |
#db_conn ⇒ Object
111 112 113 114 115 116 117 118 119 |
# File 'lib/flatfish/pleuronectiformes.rb', line 111 def db_conn ActiveRecord::Base.establish_connection( :adapter=> "mysql2", :host => "localhost", :username => @config['db_user'], :password => @config['db_pass'], :database=> @config['db'] ) end |
#ingurgitate ⇒ Object
main loop for flatfish
16 17 18 19 20 21 22 23 24 25 26 27 |
# File 'lib/flatfish/pleuronectiformes.rb', line 16 def ingurgitate create_media unless Flatfish::Media.table_exists? @config["types"].each do |k,v| next if v["csv"].nil? @csv_file = v["csv"] @host = v["host"] create_klass(k) parse(k) end output_schema end |
#load_csv ⇒ Object
load csv, set schema
50 51 52 53 54 |
# File 'lib/flatfish/pleuronectiformes.rb', line 50 def load_csv csv = CSV.read(@csv_file) @schema = csv.shift[3..-1] return csv end |
#output_schema ⇒ Object
generate a dynamic schema.yml for Migrate mapping
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
# File 'lib/flatfish/pleuronectiformes.rb', line 81 def output_schema # TODO REFACTOR THIS ISH klasses = @klasses File.open('schema.yml', 'w') do |out| output = Hash.new output["schema"] = Hash.new klasses["Media"] = Flatfish::Media klasses.each_pair do |k,v| klass_attributes = Hash.new v.new.attributes.each { |a| klass_attributes[a[0]] = split_type(v.columns_hash[a[0]].sql_type) } output["schema"].merge!({k => {"machine_name" => k.tableize, "fields" => klass_attributes, "primary key" => ["id"]}}) end out.write output.to_yaml end end |
#parse(k) ⇒ Object
loop thru csv
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
# File 'lib/flatfish/pleuronectiformes.rb', line 57 def parse(k) csv = load_csv @cnt = 0 csv.each do |row| begin break if @cnt == @config['max_rows'] @cnt += 1 page = @klass.find_or_create_by_url(row[0]) puts "Processing #{k}.#{page.id} with URL #{row[0]}" page.setup(row, @config, @schema, @host) page.process page.save! rescue Exception => e if e. =~ /(redirection forbidden|404 Not Found)/ ap "URL: #{page.url} #{e}" else ap "URL: #{page.url} ERROR: #{e} BACKTRACE: #{e.backtrace}" end end end end |
#split_type(type) ⇒ Object
helper function to convert AR sql_type to Drupal format; eg :type => varchar(255) to :type => varchar, :length => 255
101 102 103 104 105 106 107 108 |
# File 'lib/flatfish/pleuronectiformes.rb', line 101 def split_type type if type =~ /\(/ then x = type.split("(") return {"type" => x[0], "length" => x[1].sub(")","").to_i } else return {"type" => type} end end |