Class: Datapimp::Sources::Base

Inherits:
Object
  • Object
show all
Defined in:
lib/datapimp/sources.rb

Direct Known Subclasses

GithubRepository, GoogleSpreadsheet, Keen, Pivotal

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, options = {}) ⇒ Base

Returns a new instance of Base.



19
20
21
22
23
24
25
26
27
28
 
# File 'lib/datapimp/sources.rb', line 19

def initialize(name, options={})
  @name     ||= name
  @options  ||= options
  @format   ||= options.fetch(:format, :json)
  @path     ||= options.fetch(:path) { Pathname(Dir.pwd()) }

  @slug_column = options.fetch(:slug_column, :_id)

  ensure_valid_options!
end

Class Attribute Details

.required_options ⇒ Object

Returns the value of attribute required_options.



12
13
14
 
# File 'lib/datapimp/sources.rb', line 12

def required_options
  @required_options
end

Instance Attribute Details

#format ⇒ Object

Returns the value of attribute format.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def format
  @format
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



8
9
10
 
# File 'lib/datapimp/sources.rb', line 8

def name
  @name
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



8
9
10
 
# File 'lib/datapimp/sources.rb', line 8

def options
  @options
end

#path ⇒ Object

Returns the value of attribute path.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def path
  @path
end

#processed ⇒ Object

Returns the value of attribute processed.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def processed
  @processed
end

#raw ⇒ Object

Returns the value of attribute raw.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def raw
  @raw
end

#refreshed_at ⇒ Object

Returns the value of attribute refreshed_at.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def refreshed_at
  @refreshed_at
end

#scopes ⇒ Object

Returns the value of attribute scopes.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def scopes
  @scopes
end

#slug_column ⇒ Object

Returns the value of attribute slug_column.



9
10
11
 
# File 'lib/datapimp/sources.rb', line 9

def slug_column
  @slug_column
end

Class Method Details

.requires(*args) ⇒ Object 



15
16
17
 
# File 'lib/datapimp/sources.rb', line 15

def self.requires *args
  self.required_options = args
end

Instance Method Details

#age ⇒ Object

how long since this data source has been refreshed?



129
130
131
 
# File 'lib/datapimp/sources.rb', line 129

def age
  Time.now.to_i - refreshed_at.to_i
end

#compute_properties ⇒ Object

compute properties takes the raw data of each record and sets additional properties on the records which may not be persited in the data source



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/datapimp/sources.rb', line 56

def compute_properties
  self.processed && self.processed.map! do |row|
    if slug_column && row.respond_to?(slug_column)
      row.slug = row.send(slug_column).to_s.parameterize
    end

    row
  end

  processors.each do |processor|
    original = self.processed.dup
    modified = []

    original.each_with_index do |record, index|
      previous = original[index - 1]
      modified.push(processor.call(record, index, previous: previous, set: original))
    end

    self.processed = modified
  end
end

#data ⇒ Object 



133
134
135
136
 
# File 'lib/datapimp/sources.rb', line 133

def data
  refresh if need_to_refresh?
  processed
end

#ensure_valid_options! ⇒ Object

makes sure that the required options for this data source are passed for any instance of the data source



86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/datapimp/sources.rb', line 86

def ensure_valid_options!
  missing_options = (Array(self.class.required_options) - options.keys.map(&:to_sym))

  missing_options.reject! do |key|
    respond_to?(key) && !send(key).nil?
  end

  if missing_options.length > 0
    raise 'Error: failure to supply the following options: ' + missing_options.map(&:to_s).join(",")
  end
end

#fetch ⇒ Object 



147
148
149
150
 
# File 'lib/datapimp/sources.rb', line 147

def fetch
  @fetched = true
  self.raw = []
end

#file ⇒ Object 



184
185
186
187
188
189
 
# File 'lib/datapimp/sources.rb', line 184

def file
  @file ||= name.parameterize if name.respond_to?(:parameterize)
  @file.gsub!("-","_")
  @file = "#{@file}.json" unless @file.match(/\.json/i)
  @file
end

#fresh_on_server? ⇒ Boolean

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/datapimp/sources.rb', line 119

def fresh_on_server?
  need_to_refresh?
end

#has_scope?(scope_name) ⇒ Boolean

Returns:

  • (Boolean) 


49
50
51
 
# File 'lib/datapimp/sources.rb', line 49

def has_scope?(scope_name)
  scope_name && (self.scopes ||= {}).key?(scope_name.to_sym)
end

#jsonify(value) ⇒ Object 



195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/datapimp/sources.rb', line 195

def jsonify(value)
  case value
  when String, Numeric, NilClass, TrueClass, FalseClass
    value
  when Hash
    Hash[value.map { |k, v| [jsonify(k), jsonify(v)] }]
  when Array
    value.map { |v| jsonify(v) }
  when HappyMapper
    value.instance_variables.each_with_object({}) do |var_name, memo|
      key       = var_name.to_s.sub(/^@/, '').to_sym
      val       = value.instance_variable_get(var_name)
      memo[key] = jsonify(val)
    end
  else
    if value.respond_to?(:to_attrs)
      value.to_attrs
    elsif value.respond_to?(:as_json)
      value.as_json
    else
      value.to_s
    end
  end
end

#max_age ⇒ Object 



123
124
125
126
 
# File 'lib/datapimp/sources.rb', line 123

def max_age
  max = ENV['MAX_DATA_SOURCE_AGE']
  (max && max.to_i) || 120
end

#need_to_refresh? ⇒ Boolean

Returns:

  • (Boolean) 


143
144
145
 
# File 'lib/datapimp/sources.rb', line 143

def need_to_refresh?
  !(@fetched && @_processed)
end

#path_to_file ⇒ Object 



191
192
193
 
# File 'lib/datapimp/sources.rb', line 191

def path_to_file
  Pathname(path).join("#{ file }")
end

#persisted? ⇒ Boolean

Returns:

  • (Boolean) 


180
181
182
 
# File 'lib/datapimp/sources.rb', line 180

def persisted?
  path_to_file && path_to_file.exist?
end

#preprocess ⇒ Object 



152
153
154
 
# File 'lib/datapimp/sources.rb', line 152

def preprocess
  self.raw.dup
end

#process ⇒ Object 



156
157
158
159
160
161
162
 
# File 'lib/datapimp/sources.rb', line 156

def process
  @_processed = true
  self.processed = preprocess
  # set_id
  compute_properties
  self.processed
end

#processors(&block) ⇒ Object 



78
79
80
81
82
 
# File 'lib/datapimp/sources.rb', line 78

def processors &block
  @processors ||= []
  @processors << block if block_given?
  @processors
end

#refresh ⇒ Object 



102
103
104
105
106
107
 
# File 'lib/datapimp/sources.rb', line 102

def refresh
  fetch
  process
  self.refreshed_at = Time.now.to_i
  self
end

#refresh! ⇒ Object 



138
139
140
141
 
# File 'lib/datapimp/sources.rb', line 138

def refresh!
  refresh
  save_to_disk
end

#refresh_if_stale? ⇒ Boolean

Returns:

  • (Boolean) 


109
110
111
 
# File 'lib/datapimp/sources.rb', line 109

def refresh_if_stale?
  refresh! if stale?
end

#save_to_disk ⇒ Object 



172
173
174
175
176
177
178
 
# File 'lib/datapimp/sources.rb', line 172

def save_to_disk
  unless path_to_file.dirname.exist?
    FileUtils.mkdir(path_to_file.dirname)
  end

  path_to_file.open('w+') {|fh| fh.write(to_s) }
end

#scope(*args, block) ⇒ Object

defines a scope for the records in this data source a scope is a named filter, implemented in the form of a block which is passed each record. if the block returns true, it returns the record:

Example:

data_source(:galleries) do

scope :active, -> {|record| record.state == "active" }

end



44
45
46
47
 
# File 'lib/datapimp/sources.rb', line 44

def scope(*args, block)
  name = args.first
  (self.scopes ||= {})[name.to_sym] = block
end

#select(&block) ⇒ Object 



98
99
100
 
# File 'lib/datapimp/sources.rb', line 98

def select(&block)
  data.select(&block)
end

#stale? ⇒ Boolean

A data source is stale if it has been populated and the age is greater than the max age we allow.

Returns:

  • (Boolean) 


115
116
117
 
# File 'lib/datapimp/sources.rb', line 115

def stale?
  !need_to_refresh? && (age > max_age)
end

#to_s ⇒ Object 



30
31
32
 
# File 'lib/datapimp/sources.rb', line 30

def to_s
  data.to_json
end