Class: Flute::SpiderBase

Inherits:
Object
  • Object
show all
Defined in:
lib/flute/spider_base.rb

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.middleware(value) ⇒ Object 



73
74
75
 
# File 'lib/flute/spider_base.rb', line 73

def middleware(value)
  opts[:middleware] = value
end

.opts ⇒ Object 



58
59
60
61
62
63
 
# File 'lib/flute/spider_base.rb', line 58

def opts
  @opts ||= {
    middleware: [:before_parse, :init_queue, :parse, :run_queue, :after_parse],
    network: Network.new
  }
end

.set_name(value) ⇒ Object 



65
66
67
 
# File 'lib/flute/spider_base.rb', line 65

def set_name(value)
  opts[:name] = value
end

.set_network(value) ⇒ Object 



77
78
79
 
# File 'lib/flute/spider_base.rb', line 77

def set_network(value)
  opts[:network] = value
end

.start_urls(value) ⇒ Object 



69
70
71
 
# File 'lib/flute/spider_base.rb', line 69

def start_urls(value)
  opts[:start_urls] = value
end

Instance Method Details

#after_parse(_requests) ⇒ Object 



42
43
44
 
# File 'lib/flute/spider_base.rb', line 42

def after_parse(_requests)
  _requests
end

#before_parse(_items) ⇒ Object 



24
25
26
 
# File 'lib/flute/spider_base.rb', line 24

def before_parse(_items)
  _items
end

#init_queue(_items) ⇒ Object 



28
29
30
31
 
# File 'lib/flute/spider_base.rb', line 28

def init_queue(_items)
  reset_manager
  _items
end

#items ⇒ Object 



20
21
22
 
# File 'lib/flute/spider_base.rb', line 20

def items
  []
end

#manager ⇒ Object 



12
13
14
 
# File 'lib/flute/spider_base.rb', line 12

def manager
  @manager
end

#manager_network ⇒ Object 



4
5
6
 
# File 'lib/flute/spider_base.rb', line 4

def manager_network
  network
end

#network ⇒ Object 



8
9
10
 
# File 'lib/flute/spider_base.rb', line 8

def network
  self.class.opts[:network]
end

#parse(_items) ⇒ Object 



33
34
35
 
# File 'lib/flute/spider_base.rb', line 33

def parse(_items)
  _items
end

#pipeline ⇒ Object 



46
47
48
49
50
51
52
53
 
# File 'lib/flute/spider_base.rb', line 46

def pipeline
  methods = self.class.opts[:middleware].map {|name| method(name)}
  _items = items
  methods.each { |middleware|
    _items = middleware.call(_items)
  }
  _items
end

#reset_manager ⇒ Object 



16
17
18
 
# File 'lib/flute/spider_base.rb', line 16

def reset_manager
  @manager = manager_network.create_manager
end

#run_queue(_items) ⇒ Object 



37
38
39
40
 
# File 'lib/flute/spider_base.rb', line 37

def run_queue(_items)
  manager.run if manager.queued_requests.size > 0
  _items
end