Class: Retriever::FetchSitemap

Inherits:
Fetch
  • Object
show all
Defined in:
lib/retriever/fetchsitemap.rb

Constant Summary

Constants inherited from Fetch

Retriever::Fetch::HR

Instance Attribute Summary

Attributes inherited from Fetch

#max_pages, #result, #t

Instance Method Summary collapse

Methods inherited from Fetch

#dump, #errlog, #good_response?, #lg, #start, #write

Constructor Details

#initialize(url, options) ⇒ FetchSitemap

recieves target URL and RR options returns an array of all unique pages found on the site



6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/retriever/fetchsitemap.rb', line 6

def initialize(url, options)
  super
  start
  @result.push(@t.target)
  @result.concat(@link_stack)

  async_crawl_and_collect
  # done, make sure progress bar says we are done
  @progressbar.finish if @progress
  @result.sort_by! { |x| x.length } if @result.size > 1
  @result.uniq!
end