Class: InstDataShipper::Destinations::S3

Inherits:
Base
  • Object
show all
Includes:
Concerns::Chunking
Defined in:
lib/inst_data_shipper/destinations/s3.rb

Constant Summary

Constants included from Concerns::Chunking

Concerns::Chunking::DEFAULT_CHUNK_SIZE

Instance Attribute Summary

Attributes inherited from Base

#dumper

Instance Method Summary collapse

Methods included from Concerns::Chunking

#group_key

Methods inherited from Base

#cleanup_fatal_error, #config, #finalize_dump, #group_key, #initialize, #initialize_dump, #preinitialize_dump, #user_config

Constructor Details

This class inherits a constructor from InstDataShipper::Destinations::Base

Instance Method Details

#chunk_data(generator, table:, extra: nil) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/inst_data_shipper/destinations/s3.rb', line 6

def chunk_data(generator, table:, extra: nil)
  warehouse_name = table[:warehouse_name]

  super(generator) do |batch, idx|
    bits = [warehouse_name, extra, idx].compact
    temp_file = "#{working_dir}/#{bits.join('.')}.csv"

    CSV.open(temp_file, 'w', headers: false) do |row|
      row << table[:columns].map { |c| c[:warehouse_name] }
      batch.each do |batch_row|
        row << batch_row
      end
    end

    yield temp_file

    File.delete(temp_file)
  end
end

#upload_data_chunk(table_def, chunk) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/inst_data_shipper/destinations/s3.rb', line 26

def upload_data_chunk(table_def, chunk)
  s3 = Aws::S3::Resource.new(client: aws_client)
  dir_key = tracker.created_at.strftime("%Y-%m-%dT%H:%M") + "_#{tracker.id}"
  bucket = s3.bucket(config[:bucket])

  subpath = config[:path].presence || "/"
  subpath = subpath[1..-1] if subpath.starts_with?("/")
  subpath = "instructure" unless subpath.present?

  obj_path = File.join(config[:path], dir_key, File.basename(chunk))
  object = bucket.object(obj_path)

  File.open(chunk, 'rb') do |file|
    object.put(body: file)
  end
end