Class: Proto::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/proto/scraper.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url) ⇒ Scraper

Returns a new instance of Scraper.



5
6
7
8
 
# File 'lib/proto/scraper.rb', line 5

def initialize(url)
  @url = url.chomp '/'
  @doc = Nokogiri::HTML(open(url))
end

Instance Attribute Details

#doc ⇒ Object

Returns the value of attribute doc.



3
4
5
 
# File 'lib/proto/scraper.rb', line 3

def doc
  @doc
end

#url ⇒ Object

Returns the value of attribute url.



3
4
5
 
# File 'lib/proto/scraper.rb', line 3

def url
  @url
end

#url_collection ⇒ Object

Returns the value of attribute url_collection.



3
4
5
 
# File 'lib/proto/scraper.rb', line 3

def url_collection
  @url_collection
end

Instance Method Details

#collect_urls(selector) ⇒ Object 



10
11
12
13
14
 
# File 'lib/proto/scraper.rb', line 10

def collect_urls(selector)
  @url_collection = doc.css(selector).map do |link|
    "#{url}#{link['href']}"
  end
end

#fetch(name = 'Type', args) ⇒ Object Also known as: fetch_and_create! 



16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/proto/scraper.rb', line 16

def fetch(name='Type', args)
  if url_collection
    attributes = visit_urls_and_fetch(args)
    protos     = create_return_objects(name, attributes)
    return protos        
  else
    attributes = scrape_attribute_data(args)
    protos     = create_return_objects(name, attributes)
    return protos
  end
end