Module: WebScraper

Defined in:
lib/web_scraper.rb

Overview

Web Scraper

Instance Method Summary collapse

Instance Method Details

#download(params) ⇒ Object

download method



21
22
23
24
25
 
# File 'lib/web_scraper.rb', line 21

def download(params)
  @logger.info("downloading... #{self.url(params[:uri])}")
  @logger.debug(" with params... #{params[:params]}")    
  @last_page = @client.post(self.url(params[:uri]), :body => params[:params], :follow_redirect => true).body
end

#scrape(options) ⇒ Object 



27
28
29
 
# File 'lib/web_scraper.rb', line 27

def scrape(options)
  ScrapedResponse.new(@last_page, options)    
end

#url(uri) ⇒ Object 



31
32
33
 
# File 'lib/web_scraper.rb', line 31

def url(uri)
  "#{@base_url}#{uri}"
end

#web_page(base_url, options = {}, &block) ⇒ Object

web_page is the method that will start the download type description options



12
13
14
15
16
17
18
 
# File 'lib/web_scraper.rb', line 12

def web_page(base_url, options = {} ,&block)    
  @logger            = LoggerConfigurer.create_logger(options)
  @client            = HTTPClient.new
  @base_url          = base_url  

  block_given? ? self.instance_eval(&block) : self.download(:uri => '/')
end