Class: Crawler

Inherits:
Object
  • Object
show all
Includes:
Process
Defined in:
lib/hopi/crawler.rb

Instance Method Summary collapse

Constructor Details

#initialize(crawler_count = 1) ⇒ Crawler

Returns a new instance of Crawler.



5
6
7
 
# File 'lib/hopi/crawler.rb', line 5

def initialize crawler_count = 1
  @crawler_count = crawler_count
end

Instance Method Details

#pooling_crawl(crawl_target) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/hopi/crawler.rb', line 9

def pooling_crawl crawl_target

  pool_size = crawl_target.count / @crawler_count
  pooled_crawl_target = crawl_target.each_slice pool_size

  pooled_crawl_target.to_a.map do |pool|
    Thread.new do
      pool.map do |pool_unit|
        yield pool_unit
      end
    end
  end.map(&:join).flatten
end