Class: PageByPage::Fetch

Inherits:
Object
  • Object
show all
Includes:
Common
Defined in:
lib/page_by_page/fetch.rb

Instance Method Summary collapse

Methods included from Common

#header, #interval, #no_progress, #selector, #to

Constructor Details

#initialize(opt = {}, &block) ⇒ Fetch

Returns a new instance of Fetch.



11
12
13
14
15
16
 
# File 'lib/page_by_page/fetch.rb', line 11

def initialize(opt = {}, &block)
  @from, @step, @to = 1, 1, Float::INFINITY
  super
  @enum = Enum.new(enum_options)
  @enum = MutexEnum.new(@enum) if defined? @threads
end

Instance Method Details

#enumerator(e) ⇒ Object 



34
35
36
 
# File 'lib/page_by_page/fetch.rb', line 34

def enumerator e
  @enumerator = e
end

#from(n) ⇒ Object 



22
23
24
 
# File 'lib/page_by_page/fetch.rb', line 22

def from n
  @from = n
end

#iterator ⇒ Object 



47
48
49
50
51
52
53
54
55
 
# File 'lib/page_by_page/fetch.rb', line 47

def iterator
  Enumerator.new do |yielder|
    items_enum.each do |_, items|
      items.each do |i|
        yielder.yield(i)
      end
    end
  end
end

#process ⇒ Object 



38
39
40
41
42
43
44
45
 
# File 'lib/page_by_page/fetch.rb', line 38

def process
  nodes_2d = defined?(@threads) ? parallel_fetch : _fetch
  puts if @progress

  nodes_2d.sort.each_with_object([]) do |key_items, res|
    res.concat key_items[1] unless key_items[1].nil?
  end
end

#step(n) ⇒ Object 



26
27
28
 
# File 'lib/page_by_page/fetch.rb', line 26

def step n
  @step = n
end

#threads(n) ⇒ Object 



30
31
32
 
# File 'lib/page_by_page/fetch.rb', line 30

def threads n
  @threads = n
end

#url(tmpl) ⇒ Object 



18
19
20
 
# File 'lib/page_by_page/fetch.rb', line 18

def url tmpl
  @tmpl = ERB.new tmpl
end