Class: WebCrawler::BatchRequest

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
lib/web_crawler/batch_request.rb

Overview

Usage:

BatchRequest.new(urls).process #=> array of Responses

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*urls) ⇒ BatchRequest

Returns a new instance of BatchRequest.



15
16
17
18
19
20
21
 
# File 'lib/web_crawler/batch_request.rb', line 15

def initialize(*urls)
  @options = urls.last.is_a?(Hash) ? urls.pop : { }
  set_handler

  @urls, @requests = urls.flatten, []
  init_requests!
end

Instance Attribute Details

#requests ⇒ Object

Returns the value of attribute requests.



10
11
12
 
# File 'lib/web_crawler/batch_request.rb', line 10

def requests
  @requests
end

#responses ⇒ Object

Returns the value of attribute responses.



10
11
12
 
# File 'lib/web_crawler/batch_request.rb', line 10

def responses
  @responses
end

#urls ⇒ Object (readonly)

Returns the value of attribute urls.



10
11
12
 
# File 'lib/web_crawler/batch_request.rb', line 10

def urls
  @urls
end

Instance Method Details

#build_request(url) ⇒ Object 



51
52
53
 
# File 'lib/web_crawler/batch_request.rb', line 51

def build_request(url)
  request_class.new(url)
end

#each(&block) ⇒ Object 



35
36
37
38
39
40
41
 
# File 'lib/web_crawler/batch_request.rb', line 35

def each &block
  @responses = []
  requests.each do |req|
    @responses << req.process
    block.call(@responses.last)
  end
end

#process ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/web_crawler/batch_request.rb', line 23

def process
  if @handler
    block_given? ? yield(@handler.process) : @handler.process
  else
    ready = requests.select{|r| r.ready? }
    @responses ||= Parallel.map(requests - ready) do |req|
      WebCrawler.logger.info "start request to #{req.url.to_s}"
      block_given? ? yield(req.process) : req.process
    end.compact + ready.map(&:process)
  end
end

#response ⇒ Object 



47
48
49
 
# File 'lib/web_crawler/batch_request.rb', line 47

def response
  responses.first
end