Class: Processor::Data::BatchProcessor

Inherits:
NullProcessor show all
Defined in:
lib/processor/data/batch_processor.rb

Overview

TODO: Change it to be useful. Usually the external iterator is provided and #results are mapped to it

Direct Known Subclasses

SolrPagesProcessor

Instance Method Summary collapse

Methods inherited from NullProcessor

#error, #finalize, #finish, #process, #record_error, #start

Constructor Details

#initialize(batch_size = 10) ⇒ BatchProcessor

Returns a new instance of BatchProcessor.



8
9
10
 
# File 'lib/processor/data/batch_processor.rb', line 8

def initialize(batch_size = 10)
  @batch_size = batch_size
end

Instance Method Details

#fetch_batch ⇒ Object 



22
23
24
25
26
27
28
 
# File 'lib/processor/data/batch_processor.rb', line 22

def fetch_batch
  @fetcher ||= query.each_slice(batch_size)
  # TODO get rid of .next enumeration here
  @fetcher.next
rescue StopIteration
  []
end

#query ⇒ Object

Raises:

  • (NotImplementedError) 


34
35
36
 
# File 'lib/processor/data/batch_processor.rb', line 34

def query
  raise NotImplementedError
end

#records ⇒ Object 



12
13
14
15
16
17
18
19
20
 
# File 'lib/processor/data/batch_processor.rb', line 12

def records
  Enumerator.new do |result|
    while (batch = fetch_batch).any?
      batch.each do |record|
        result << record
      end
    end
  end
end

#total_records ⇒ Object 



30
31
32
 
# File 'lib/processor/data/batch_processor.rb', line 30

def total_records
  @total_records ||= query.count
end