Module: Outlander::Crawler

Defined in:
lib/outlander/crawler.rb

Defined Under Namespace

Modules: ClassMethods

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  num_threads: 3,
  pause: 1
}

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.included(base) ⇒ Object 



16
17
18
19
20
21
22
23
 
# File 'lib/outlander/crawler.rb', line 16

def included(base)
  base.extend ClassMethods

  base.class_eval do
    @roots = {}
    @handlers = {}
  end
end

Instance Method Details

#initialize(options = {}) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
 
# File 'lib/outlander/crawler.rb', line 47

def initialize(options = {})
  @cache = options.delete(:cache)
  @options = DEFAULT_OPTIONS.merge options
  @history = {}
  @mutex = Mutex.new
  @pool = ThreadsPool.new @options[:num_threads]
  self.class.roots.each do |url, handler|
    discover url, handler
  end
end

#run!(&block) ⇒ Object 



58
59
60
61
62
 
# File 'lib/outlander/crawler.rb', line 58

def run!(&block)
  @result_handler = block
  instance_eval &self.class.setup
  @pool.start
end