Class: Spieker::Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/spieker/crawler.rb

Instance Method Summary collapse

Constructor Details

#initialize(url, verbose: false, lang: 'en') ⇒ Crawler

Returns a new instance of Crawler.



3
4
5
6
7
8
 
# File 'lib/spieker/crawler.rb', line 3

def initialize(url, verbose: false, lang: 'en')
  @url = url
  @tracked_links = []
  @verbose = verbose
  @lang = lang
end

Instance Method Details

#crawl! ⇒ Object 



10
11
12
13
14
15
16
17
18
19
 
# File 'lib/spieker/crawler.rb', line 10

def crawl!
  report "Starting to crawl on #{@url}"

  scraper = LinkScraper.new(@url, lang: @lang)
  track_link(@url)
  links =  scraper.result
  recursively_crawl(links)

  print_results if @verbose
end

#current_path ⇒ Object 



21
22
23
 
# File 'lib/spieker/crawler.rb', line 21

def current_path
  URI.parse(@url).path
end