Class: Datapimp::Sources::Base
- Inherits:
-
Object
- Object
- Datapimp::Sources::Base
- Defined in:
- lib/datapimp/sources.rb
Direct Known Subclasses
Class Attribute Summary collapse
-
.required_options ⇒ Object
Returns the value of attribute required_options.
Instance Attribute Summary collapse
-
#format ⇒ Object
Returns the value of attribute format.
-
#name ⇒ Object
readonly
Returns the value of attribute name.
-
#options ⇒ Object
readonly
Returns the value of attribute options.
-
#path ⇒ Object
Returns the value of attribute path.
-
#processed ⇒ Object
Returns the value of attribute processed.
-
#raw ⇒ Object
Returns the value of attribute raw.
-
#refreshed_at ⇒ Object
Returns the value of attribute refreshed_at.
-
#scopes ⇒ Object
Returns the value of attribute scopes.
-
#slug_column ⇒ Object
Returns the value of attribute slug_column.
Class Method Summary collapse
Instance Method Summary collapse
-
#age ⇒ Object
how long since this data source has been refreshed?.
-
#compute_properties ⇒ Object
compute properties takes the raw data of each record and sets additional properties on the records which may not be persited in the data source.
- #data ⇒ Object
-
#ensure_valid_options! ⇒ Object
makes sure that the required options for this data source are passed for any instance of the data source.
- #fetch ⇒ Object
- #file ⇒ Object
- #fresh_on_server? ⇒ Boolean
- #has_scope?(scope_name) ⇒ Boolean
-
#initialize(name, options = {}) ⇒ Base
constructor
A new instance of Base.
- #max_age ⇒ Object
- #need_to_refresh? ⇒ Boolean
- #path_to_file ⇒ Object
- #persisted? ⇒ Boolean
- #preprocess ⇒ Object
- #process ⇒ Object
- #processors(&block) ⇒ Object
- #refresh ⇒ Object
- #refresh! ⇒ Object
- #refresh_if_stale? ⇒ Boolean
- #save_to_disk ⇒ Object
-
#scope(*args, block) ⇒ Object
defines a scope for the records in this data source a scope is a named filter, implemented in the form of a block which is passed each record.
- #select(&block) ⇒ Object
-
#stale? ⇒ Boolean
A data source is stale if it has been populated and the age is greater than the max age we allow.
- #to_s ⇒ Object
Constructor Details
#initialize(name, options = {}) ⇒ Base
Returns a new instance of Base.
19 20 21 22 23 24 25 26 27 28 |
# File 'lib/datapimp/sources.rb', line 19 def initialize(name, ={}) @name ||= name @options ||= @format ||= .fetch(:format, :json) @path ||= .fetch(:path) { Pathname(Dir.pwd()) } @slug_column = .fetch(:slug_column, :_id) end |
Class Attribute Details
.required_options ⇒ Object
Returns the value of attribute required_options.
12 13 14 |
# File 'lib/datapimp/sources.rb', line 12 def @required_options end |
Instance Attribute Details
#format ⇒ Object
Returns the value of attribute format.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def format @format end |
#name ⇒ Object (readonly)
Returns the value of attribute name.
8 9 10 |
# File 'lib/datapimp/sources.rb', line 8 def name @name end |
#options ⇒ Object (readonly)
Returns the value of attribute options.
8 9 10 |
# File 'lib/datapimp/sources.rb', line 8 def @options end |
#path ⇒ Object
Returns the value of attribute path.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def path @path end |
#processed ⇒ Object
Returns the value of attribute processed.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def processed @processed end |
#raw ⇒ Object
Returns the value of attribute raw.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def raw @raw end |
#refreshed_at ⇒ Object
Returns the value of attribute refreshed_at.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def refreshed_at @refreshed_at end |
#scopes ⇒ Object
Returns the value of attribute scopes.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def scopes @scopes end |
#slug_column ⇒ Object
Returns the value of attribute slug_column.
9 10 11 |
# File 'lib/datapimp/sources.rb', line 9 def slug_column @slug_column end |
Class Method Details
.requires(*args) ⇒ Object
15 16 17 |
# File 'lib/datapimp/sources.rb', line 15 def self.requires *args self. = args end |
Instance Method Details
#age ⇒ Object
how long since this data source has been refreshed?
129 130 131 |
# File 'lib/datapimp/sources.rb', line 129 def age Time.now.to_i - refreshed_at.to_i end |
#compute_properties ⇒ Object
compute properties takes the raw data of each record and sets additional properties on the records which may not be persited in the data source
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
# File 'lib/datapimp/sources.rb', line 56 def compute_properties self.processed && self.processed.map! do |row| if slug_column && row.respond_to?(slug_column) row.slug = row.send(slug_column).to_s.parameterize end row end processors.each do |processor| original = self.processed.dup modified = [] original.each_with_index do |record, index| previous = original[index - 1] modified.push(processor.call(record, index, previous: previous, set: original)) end self.processed = modified end end |
#data ⇒ Object
133 134 135 136 |
# File 'lib/datapimp/sources.rb', line 133 def data refresh if need_to_refresh? processed end |
#ensure_valid_options! ⇒ Object
makes sure that the required options for this data source are passed for any instance of the data source
86 87 88 89 90 91 92 93 94 95 96 |
# File 'lib/datapimp/sources.rb', line 86 def = (Array(self.class.) - .keys.map(&:to_sym)) .reject! do |key| respond_to?(key) && !send(key).nil? end if .length > 0 raise 'Error: failure to supply the following options: ' + .map(&:to_s).join(",") end end |
#fetch ⇒ Object
147 148 149 150 |
# File 'lib/datapimp/sources.rb', line 147 def fetch @fetched = true self.raw = [] end |
#file ⇒ Object
184 185 186 187 188 189 |
# File 'lib/datapimp/sources.rb', line 184 def file @file ||= name.parameterize if name.respond_to?(:parameterize) @file.gsub!("-","_") @file = "#{@file}.json" unless @file.match(/\.json/i) @file end |
#fresh_on_server? ⇒ Boolean
119 120 121 |
# File 'lib/datapimp/sources.rb', line 119 def fresh_on_server? need_to_refresh? end |
#has_scope?(scope_name) ⇒ Boolean
49 50 51 |
# File 'lib/datapimp/sources.rb', line 49 def has_scope?(scope_name) scope_name && (self.scopes ||= {}).key?(scope_name.to_sym) end |
#max_age ⇒ Object
123 124 125 126 |
# File 'lib/datapimp/sources.rb', line 123 def max_age max = ENV['MAX_DATA_SOURCE_AGE'] (max && max.to_i) || 120 end |
#need_to_refresh? ⇒ Boolean
143 144 145 |
# File 'lib/datapimp/sources.rb', line 143 def need_to_refresh? !(@fetched && @_processed) end |
#path_to_file ⇒ Object
191 192 193 |
# File 'lib/datapimp/sources.rb', line 191 def path_to_file Pathname(path).join("#{ file }") end |
#persisted? ⇒ Boolean
180 181 182 |
# File 'lib/datapimp/sources.rb', line 180 def persisted? path_to_file && path_to_file.exist? end |
#preprocess ⇒ Object
152 153 154 |
# File 'lib/datapimp/sources.rb', line 152 def preprocess self.raw.dup end |
#process ⇒ Object
156 157 158 159 160 161 162 |
# File 'lib/datapimp/sources.rb', line 156 def process @_processed = true self.processed = preprocess # set_id compute_properties self.processed end |
#processors(&block) ⇒ Object
78 79 80 81 82 |
# File 'lib/datapimp/sources.rb', line 78 def processors &block @processors ||= [] @processors << block if block_given? @processors end |
#refresh ⇒ Object
102 103 104 105 106 107 |
# File 'lib/datapimp/sources.rb', line 102 def refresh fetch process self.refreshed_at = Time.now.to_i self end |
#refresh! ⇒ Object
138 139 140 141 |
# File 'lib/datapimp/sources.rb', line 138 def refresh! refresh save_to_disk end |
#refresh_if_stale? ⇒ Boolean
109 110 111 |
# File 'lib/datapimp/sources.rb', line 109 def refresh_if_stale? refresh! if stale? end |
#save_to_disk ⇒ Object
172 173 174 175 176 177 178 |
# File 'lib/datapimp/sources.rb', line 172 def save_to_disk unless path_to_file.dirname.exist? FileUtils.mkdir(path_to_file.dirname) end path_to_file.open('w+') {|fh| fh.write(to_s) } end |
#scope(*args, block) ⇒ Object
defines a scope for the records in this data source a scope is a named filter, implemented in the form of a block which is passed each record. if the block returns true, it returns the record:
Example:
data_source(:galleries) do
scope :active, -> {|record| record.state == "active" }
end
44 45 46 47 |
# File 'lib/datapimp/sources.rb', line 44 def scope(*args, block) name = args.first (self.scopes ||= {})[name.to_sym] = block end |
#select(&block) ⇒ Object
98 99 100 |
# File 'lib/datapimp/sources.rb', line 98 def select(&block) data.select(&block) end |
#stale? ⇒ Boolean
A data source is stale if it has been populated and the age is greater than the max age we allow.
115 116 117 |
# File 'lib/datapimp/sources.rb', line 115 def stale? !need_to_refresh? && (age > max_age) end |
#to_s ⇒ Object
30 31 32 |
# File 'lib/datapimp/sources.rb', line 30 def to_s data.to_json end |