Class: OrganicSitemap::CrawlerManager

Inherits:
Object
  • Object
show all
Defined in:
lib/organic-sitemap/crawler_manager.rb

Class Method Summary collapse

Class Method Details

.warmup(urls, opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/organic-sitemap/crawler_manager.rb', line 3

def self.warmup(urls, opts = {})
  data = {urls: [*urls],
          valids: 0,
          invalids: {}}

  data[:urls].each do |uri|
    p "OrganicSitemap::CrawlerManager GET: #{uri}"
    response = HTTParty.get(url_for(uri), opts)

    url_processed = OrganicSitemap::UrlProcessor.new(response.code, response.headers, response.request)

    unless url_processed.url_from_cache_valid?
      p "OrganicSitemap::CrawlerManager REMOVING URL: #{uri} response code: #{response.code}"
      OrganicSitemap::RedisManager.remove_key(key: uri)
      if data[:invalids][response.code]
        data[:invalids][response.code] << uri
      else
        data[:invalids][response.code] = [uri]
      end
    else
      data[:valids] += 1
    end
    sleep crawler_delay if crawler_delay
  end
  data
end