Class: DataMiner::Base

Inherits:
Object
  • Object
show all
Includes:
Blockenspiel::DSL
Defined in:
lib/data_miner/base.rb

Constant Summary collapse

COMPLETE_UNIT_DEFINITIONS = 
[
  [:units],
  [:from_units, :to_units],
  [:units_field_name],
  [:units_field_name, :to_units],
  [:units_field_number],
  [:units_field_number, :to_units]
]

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(resource) ⇒ Base

Returns a new instance of Base.



7
8
9
10
11
12
 
# File 'lib/data_miner/base.rb', line 7

def initialize(resource)
  @steps = Array.new
  @resource = resource
  @step_counter = 0
  @attributes = HashWithIndifferentAccess.new
end

Instance Attribute Details

#attributes ⇒ Object

Returns the value of attribute attributes.



5
6
7
 
# File 'lib/data_miner/base.rb', line 5

def attributes
  @attributes
end

#resource ⇒ Object

Returns the value of attribute resource.



5
6
7
 
# File 'lib/data_miner/base.rb', line 5

def resource
  @resource
end

#step_counter ⇒ Object

Returns the value of attribute step_counter.



5
6
7
 
# File 'lib/data_miner/base.rb', line 5

def step_counter
  @step_counter
end

#steps ⇒ Object

Returns the value of attribute steps.



5
6
7
 
# File 'lib/data_miner/base.rb', line 5

def steps
  @steps
end

Class Method Details

.run(options = {}) ⇒ Object

Mine data. Defaults to all resource_names touched by DataMiner.

Options

  • :resource_names: array of resource (class) names to mine



181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/data_miner/base.rb', line 181

def run(options = {})
  options.symbolize_keys!
  
  resource_names.each do |resource_name|
    if options[:resource_names].blank? or options[:resource_names].include?(resource_name)
      resource_name.constantize.data_miner_base.run options
    end
  end
ensure
  RemoteTable.cleanup
end

Instance Method Details

#after_invoke ⇒ Object 



88
89
90
91
92
 
# File 'lib/data_miner/base.rb', line 88

def after_invoke
  return unless resource.table_exists?
  make_sure_unit_definitions_make_sense
  suggest_missing_column_migrations
end

#before_invoke ⇒ Object 



84
85
86
 
# File 'lib/data_miner/base.rb', line 84

def before_invoke
  
end

#import(*args, &block) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/data_miner/base.rb', line 31

def import(*args, &block)
  if args.length == 1
    description = '(no description)'
  else
    description = args.first
  end
  options = args.last
    
  step = DataMiner::Import.new self, step_counter, description, options
  Blockenspiel.invoke block, step
  steps << step
  self.step_counter += 1
end

#import_steps ⇒ Object 



80
81
82
 
# File 'lib/data_miner/base.rb', line 80

def import_steps
  steps.select { |step| step.is_a? Import }
end

#make_sure_unit_definitions_make_sense ⇒ Object 



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/data_miner/base.rb', line 103

def make_sure_unit_definitions_make_sense
  import_steps.each do |step|
    step.attributes.each do |_, attribute|
      if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
        DataMiner.log_or_raise %{

================================

You don't have a valid unit definition for #{resource.name}##{attribute.name}.

You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.map(&:to_sym).inspect }.

You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence}".

================================
        }
      end
    end
  end
end

#process(method_name_or_block_description, &block) ⇒ Object 



21
22
23
24
 
# File 'lib/data_miner/base.rb', line 21

def process(method_name_or_block_description, &block)
  steps << DataMiner::Process.new(self, step_counter, method_name_or_block_description, &block)
  self.step_counter += 1
end

#run(options = {}) ⇒ Object

Mine data for this class.



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/data_miner/base.rb', line 46

def run(options = {})
  options.symbolize_keys!
  
  return if DataMiner::Base.call_stack.include? resource.name
  DataMiner::Base.call_stack.push resource.name
  
  finished = false
  skipped = false
  if DataMiner::Run.table_exists?
    run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name, :killed => true
  else
    run = nil
    DataMiner.log_info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
  end
  resource.delete_all if options[:from_scratch]
  begin
    steps.each do |step|
      step.run run
      resource.reset_column_information
    end
    finished = true
  rescue DataMiner::Finish
    finished = true
  rescue DataMiner::Skip
    skipped = true
  ensure
    if DataMiner::Run.table_exists?
      run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false
    end
    DataMiner::Base.call_stack.clear if DataMiner::Base.call_stack.first == resource.name and !options[:preserve_call_stack_between_runs]
  end
  nil
end

#schema(create_table_options = {}, &block) ⇒ Object 



14
15
16
17
18
19
 
# File 'lib/data_miner/base.rb', line 14

def schema(create_table_options = {}, &block)
  step = DataMiner::Schema.new self, step_counter, create_table_options
  Blockenspiel.invoke block, step
  steps << step
  self.step_counter += 1
end

#suggest_missing_column_migrations ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/data_miner/base.rb', line 124

def suggest_missing_column_migrations
  missing_columns = Array.new
  
  import_steps.each do |step|
    step.attributes.each do |_, attribute|
      DataMiner.log_or_raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.end_with? '_units'
      unless resource.column_names.include? attribute.name
        missing_columns << attribute.name
      end
      if attribute.wants_units? and !resource.column_names.include?(units_column = "#{attribute.name}_units")
        missing_columns << units_column
      end
    end
  end
  missing_columns.uniq!
  if missing_columns.any?
    DataMiner.log_debug %{

================================

On #{resource}, it looks like you're missing some columns...

Please run this...

  ./script/generate migration AddMissingColumnsTo#{resource.name}

and **replace** the resulting file with this:

  class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
def self.up
#{missing_columns.map { |column_name| "      add_column :#{resource.table_name}, :#{column_name}, :#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
end

def self.down
#{missing_columns.map { |column_name| "      remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
end
  end

On the other hand, if you're working directly with create_table, this might be helpful:

#{missing_columns.map { |column_name| "t.#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }

================================
    }
  end
end

#tap(description, source, options = {}) ⇒ Object 



26
27
28
29
 
# File 'lib/data_miner/base.rb', line 26

def tap(description, source, options = {})
  steps << DataMiner::Tap.new(self, step_counter, description, source, options)
  self.step_counter += 1
end