Class: ForestAdminAgent::Utils::CsvGeneratorStream

Inherits:
Object
  • Object
show all
Defined in:
lib/forest_admin_agent/utils/csv_generator_stream.rb

Constant Summary collapse

CHUNK_SIZE = 
1000

Class Method Summary collapse

Class Method Details

.stream(header, filter, projection, list_records, limit_export_size = nil) ⇒ Enumerator

Returns Lazy enumerator that yields CSV rows.

Parameters:

  • header (Array<String>) —

    CSV header fields

  • filter (ForestAdminDatasourceToolkit::Components::Query::Filter) —

    Query filter

  • projection (ForestAdminDatasourceToolkit::Components::Query::Projection) —

    Fields to include

  • list_records (Proc) —

    A callable that fetches records: ->(batch_filter) { ... }

  • limit_export_size (Integer, nil) (defaults to: nil) —

    Maximum number of records to export

Returns:

  • (Enumerator) —

    Lazy enumerator that yields CSV rows



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/forest_admin_agent/utils/csv_generator_stream.rb', line 16

def self.stream(header, filter, projection, list_records, limit_export_size = nil)
  Enumerator.new do |yielder|
    # Yield header row first (client receives immediately)
    # Escaped like any other row: the labels are the caller's, and one carrying a comma or a
    # quote would otherwise emit a header row wider than the data under it.
    yielder << CSV.generate_line(parse_header(header, projection))

    offset = 0

    loop do
      # Fetch batch of records
      batch_filter = filter.override(
        page: ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: CHUNK_SIZE)
      )
      records = list_records.call(batch_filter)

      # Break if no more records
      break if records.empty?

      # Convert each record to CSV row and yield immediately
      records.each do |record|
        yielder << generate_row(record, projection)
      end

      # Update offset
      offset += CHUNK_SIZE

      # Check if we've reached the export limit
      break if limit_export_size && offset >= limit_export_size

      # Check if this was a partial batch (last batch)
      break if records.length < CHUNK_SIZE

      # Periodic garbage collection to prevent memory creep
      GC.start(full_mark: false) if (offset % 10_000).zero?
    end
  rescue IOError, Errno::EPIPE => e
    # Client disconnected - clean up gracefully
    Facades::Container.logger&.log(
      'Info',
      "CSV export interrupted at offset #{offset}: #{e.message}"
    )
  end
end