Class: DataMapper::YS::Scraper::Composite

Inherits:
Base
  • Object
show all
Defined in:
lib/dm-ys/scraper.rb

Overview

Composite Scraper

Instance Method Summary collapse

Methods inherited from Base

#base_uri, #count, #initialize, #register_properties!, #uri

Methods included from CachedAccessor

included

Constructor Details

This class inherits a constructor from DataMapper::YS::Scraper::Base

Instance Method Details

#labels ⇒ Object 



221
222
223
 
# File 'lib/dm-ys/scraper.rb', line 221

def labels
  pages.first.labels
end

#names ⇒ Object 



217
218
219
 
# File 'lib/dm-ys/scraper.rb', line 217

def names
  pages.first.names
end

#pages ⇒ Object 



213
214
215
 
# File 'lib/dm-ys/scraper.rb', line 213

def pages
  @pages ||= execute
end

#records ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
 
# File 'lib/dm-ys/scraper.rb', line 225

def records
  records = []
  digests = Set.new
  pages.each do |page|
    page.records.each do |entry|
      if config.uniq_entry?
        sha1 = entry.attributes.merge(:id=>nil).inspect
        next if digests.include?(sha1)
        digests << sha1
      end
      records << entry
    end
  end
  return records
end