Module: InstDataShipper::Destinations::Concerns::Chunking

Extended by:
ActiveSupport::Concern
Included in:
HostedData, S3, Speccable
Defined in:
lib/inst_data_shipper/destinations/concerns/chunking.rb

Constant Summary collapse

DEFAULT_CHUNK_SIZE = 
100_000

Instance Method Summary collapse

Instance Method Details

#chunk_data(generator, chunk_size: nil, **kwargs) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/inst_data_shipper/destinations/concerns/chunking.rb', line 9

def chunk_data(generator, chunk_size: nil, **kwargs)
  chunk_size ||= config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
  slice = 1

  btchr = CanvasSync::BatchProcessor.new(of: chunk_size) do |batch|
    yield batch, slice
    slice += 1
  end

  super(generator, **kwargs) do |row|
    btchr << row
  end

  btchr.flush
end

#group_key ⇒ Object 



25
26
27
28
29
 
# File 'lib/inst_data_shipper/destinations/concerns/chunking.rb', line 25

def group_key
  super.tap do |k|
    k[:chunk_size] = config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
  end
end